{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example of self-supervised learning using augmentation to do contrastive learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import packages and load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of leaves in the phylogeny tree after pruning:  5287\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.insert(1, '../src/')\n",
    "\n",
    "from ete4 import Tree\n",
    "from mixup import Mixup, PhylogenyDataset, PhylogenyTree\n",
    "dataset = \"ibd200\"\n",
    "target = \"type\"\n",
    "\n",
    "data_fp = f'../data/{dataset}/data.tsv.xz'\n",
    "meta_fp = f'../data/{dataset}/meta.tsv'\n",
    "target_fp = f'../data/{dataset}/{target}.py'\n",
    "\n",
    "phylogeny_tree_fp = '../data/WoL2/phylogeny.nwk'\n",
    "tree = PhylogenyTree.init_from_nwk(phylogeny_tree_fp)\n",
    "data = PhylogenyDataset.init_from_files(data_fp, meta_fp, target_fp)\n",
    "tree.prune(data.features)\n",
    "print(\"number of leaves in the phylogeny tree after pruning: \", len(list(tree.ete_tree.leaves())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get augmented data\n",
    "\n",
    "Contrastive learning requires a pair of augmented data, and we can using phylomix to generate them by specifying different random seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "data.one_hot_encode()\n",
    "mixup = Mixup(data, taxonomy_tree=None, phylogeny_tree=tree)\n",
    "idx1 = np.arange(len(data.X))\n",
    "idx2 = idx1[::-1]\n",
    "out1 = mixup.mixup(len(idx1), method='phylomix', alpha=2, tree='phylogeny', index1=idx1, index2=idx2, contrastive_learning=True, seed=0)\n",
    "out2 = mixup.mixup(len(idx1), method='phylomix', alpha=2, tree='phylogeny', index1=idx1, index2=idx2, contrastive_learning=True, seed=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define a simple model as encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderProjectionHead(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim1=1024, hidden_dim2=256, latent_dim=512, output_dim=512):\n",
    "        super(EncoderProjectionHead, self).__init__()\n",
    "        self.fc1 = nn.Linear(input_dim, input_dim // 2)\n",
    "        self.fc2 = nn.Linear(input_dim // 2, hidden_dim1)\n",
    "        self.fc3 = nn.Linear(hidden_dim1, output_dim)\n",
    "        self.encoder = nn.Sequential(\n",
    "            self.fc1,\n",
    "            nn.ReLU(),\n",
    "            self.fc2,\n",
    "            nn.ReLU(),\n",
    "            self.fc3\n",
    "        )\n",
    "        self.projection_head = nn.Sequential(\n",
    "            nn.Linear(latent_dim, 256),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(256, 128),\n",
    "            nn.BatchNorm1d(128)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        z = self.encoder(x)\n",
    "        z = self.projection_head(z)\n",
    "        return F.normalize(z, dim=1)  # Normalize the output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the simclr loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def info_nce_loss(features, batch_size, device):\n",
    "\n",
    "    labels = torch.cat([torch.arange(batch_size) for i in range(2)], dim=0).to(device)\n",
    "    labels = (labels.unsqueeze(0) == labels.unsqueeze(1)).float()\n",
    "    labels = labels\n",
    "\n",
    "    features = F.normalize(features, dim=1)\n",
    "\n",
    "    similarity_matrix = torch.matmul(features, features.T)\n",
    "\n",
    "    # discard the main diagonal from both: labels and similarities matrix\n",
    "    mask = torch.eye(labels.shape[0], dtype=torch.bool)\n",
    "    labels = labels[~mask].view(labels.shape[0], -1)\n",
    "    similarity_matrix = similarity_matrix[~mask].view(similarity_matrix.shape[0], -1)\n",
    "\n",
    "    positives = similarity_matrix[labels.bool()].view(labels.shape[0], -1)\n",
    "\n",
    "    negatives = similarity_matrix[~labels.bool()].view(similarity_matrix.shape[0], -1)\n",
    "\n",
    "    logits = torch.cat([positives, negatives], dim=1)\n",
    "    labels = torch.zeros(logits.shape[0], dtype=torch.long)\n",
    "\n",
    "    logits = logits / 0.05\n",
    "    return logits.to(device), labels.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make data into data loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "combined_data = np.stack((out1, out2), axis=1)\n",
    "combined_data = torch.tensor(combined_data, dtype=torch.float32).to(device)\n",
    "dataloader = DataLoader(combined_data, batch_size=combined_data.shape[0], shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000 - Loss: 14.777177: 100%|██████████| 1/1 [00:00<00:00,  1.48it/s]\n",
      "Epoch 2/1000 - Loss: 17.074661: 100%|██████████| 1/1 [00:00<00:00, 64.85it/s]\n",
      "Epoch 3/1000 - Loss: 17.418701: 100%|██████████| 1/1 [00:00<00:00, 61.45it/s]\n",
      "Epoch 4/1000 - Loss: 16.082857: 100%|██████████| 1/1 [00:00<00:00, 59.88it/s]\n",
      "Epoch 5/1000 - Loss: 15.644450: 100%|██████████| 1/1 [00:00<00:00, 61.66it/s]\n",
      "Epoch 6/1000 - Loss: 16.263224: 100%|██████████| 1/1 [00:00<00:00, 61.04it/s]\n",
      "Epoch 7/1000 - Loss: 16.681601: 100%|██████████| 1/1 [00:00<00:00, 61.28it/s]\n",
      "Epoch 8/1000 - Loss: 14.200163: 100%|██████████| 1/1 [00:00<00:00, 57.09it/s]\n",
      "Epoch 9/1000 - Loss: 15.402557: 100%|██████████| 1/1 [00:00<00:00, 53.22it/s]\n",
      "Epoch 10/1000 - Loss: 12.474568: 100%|██████████| 1/1 [00:00<00:00, 58.74it/s]\n",
      "Epoch 11/1000 - Loss: 12.089806: 100%|██████████| 1/1 [00:00<00:00, 58.84it/s]\n",
      "Epoch 12/1000 - Loss: 11.176358: 100%|██████████| 1/1 [00:00<00:00, 56.66it/s]\n",
      "Epoch 13/1000 - Loss: 8.757011: 100%|██████████| 1/1 [00:00<00:00, 57.86it/s]\n",
      "Epoch 14/1000 - Loss: 9.022717: 100%|██████████| 1/1 [00:00<00:00, 56.75it/s]\n",
      "Epoch 15/1000 - Loss: 8.531283: 100%|██████████| 1/1 [00:00<00:00, 61.06it/s]\n",
      "Epoch 16/1000 - Loss: 8.476146: 100%|██████████| 1/1 [00:00<00:00, 59.14it/s]\n",
      "Epoch 17/1000 - Loss: 8.157244: 100%|██████████| 1/1 [00:00<00:00, 60.68it/s]\n",
      "Epoch 18/1000 - Loss: 7.298940: 100%|██████████| 1/1 [00:00<00:00, 59.37it/s]\n",
      "Epoch 19/1000 - Loss: 8.036203: 100%|██████████| 1/1 [00:00<00:00, 60.63it/s]\n",
      "Epoch 20/1000 - Loss: 7.989518: 100%|██████████| 1/1 [00:00<00:00, 60.12it/s]\n",
      "Epoch 21/1000 - Loss: 7.984642: 100%|██████████| 1/1 [00:00<00:00, 59.57it/s]\n",
      "Epoch 22/1000 - Loss: 7.862861: 100%|██████████| 1/1 [00:00<00:00, 60.67it/s]\n",
      "Epoch 23/1000 - Loss: 7.780779: 100%|██████████| 1/1 [00:00<00:00, 58.83it/s]\n",
      "Epoch 24/1000 - Loss: 8.386421: 100%|██████████| 1/1 [00:00<00:00, 55.53it/s]\n",
      "Epoch 25/1000 - Loss: 7.722168: 100%|██████████| 1/1 [00:00<00:00, 52.82it/s]\n",
      "Epoch 26/1000 - Loss: 7.783696: 100%|██████████| 1/1 [00:00<00:00, 58.16it/s]\n",
      "Epoch 27/1000 - Loss: 7.640199: 100%|██████████| 1/1 [00:00<00:00, 58.27it/s]\n",
      "Epoch 28/1000 - Loss: 7.642972: 100%|██████████| 1/1 [00:00<00:00, 57.81it/s]\n",
      "Epoch 29/1000 - Loss: 7.859942: 100%|██████████| 1/1 [00:00<00:00, 51.78it/s]\n",
      "Epoch 30/1000 - Loss: 7.439248: 100%|██████████| 1/1 [00:00<00:00, 55.61it/s]\n",
      "Epoch 31/1000 - Loss: 7.433933: 100%|██████████| 1/1 [00:00<00:00, 58.22it/s]\n",
      "Epoch 32/1000 - Loss: 7.361742: 100%|██████████| 1/1 [00:00<00:00, 58.49it/s]\n",
      "Epoch 33/1000 - Loss: 7.325481: 100%|██████████| 1/1 [00:00<00:00, 54.76it/s]\n",
      "Epoch 34/1000 - Loss: 6.888165: 100%|██████████| 1/1 [00:00<00:00, 56.07it/s]\n",
      "Epoch 35/1000 - Loss: 7.258489: 100%|██████████| 1/1 [00:00<00:00, 57.11it/s]\n",
      "Epoch 36/1000 - Loss: 7.212502: 100%|██████████| 1/1 [00:00<00:00, 55.60it/s]\n",
      "Epoch 37/1000 - Loss: 7.205602: 100%|██████████| 1/1 [00:00<00:00, 54.48it/s]\n",
      "Epoch 38/1000 - Loss: 7.184299: 100%|██████████| 1/1 [00:00<00:00, 52.91it/s]\n",
      "Epoch 39/1000 - Loss: 7.174967: 100%|██████████| 1/1 [00:00<00:00, 58.59it/s]\n",
      "Epoch 40/1000 - Loss: 7.182088: 100%|██████████| 1/1 [00:00<00:00, 59.00it/s]\n",
      "Epoch 41/1000 - Loss: 7.161112: 100%|██████████| 1/1 [00:00<00:00, 57.55it/s]\n",
      "Epoch 42/1000 - Loss: 7.172503: 100%|██████████| 1/1 [00:00<00:00, 57.04it/s]\n",
      "Epoch 43/1000 - Loss: 7.152447: 100%|██████████| 1/1 [00:00<00:00, 59.71it/s]\n",
      "Epoch 44/1000 - Loss: 7.149529: 100%|██████████| 1/1 [00:00<00:00, 54.30it/s]\n",
      "Epoch 45/1000 - Loss: 7.133649: 100%|██████████| 1/1 [00:00<00:00, 59.59it/s]\n",
      "Epoch 46/1000 - Loss: 7.149637: 100%|██████████| 1/1 [00:00<00:00, 57.27it/s]\n",
      "Epoch 47/1000 - Loss: 7.131534: 100%|██████████| 1/1 [00:00<00:00, 60.57it/s]\n",
      "Epoch 48/1000 - Loss: 7.144103: 100%|██████████| 1/1 [00:00<00:00, 61.44it/s]\n",
      "Epoch 49/1000 - Loss: 7.140709: 100%|██████████| 1/1 [00:00<00:00, 60.39it/s]\n",
      "Epoch 50/1000 - Loss: 7.128707: 100%|██████████| 1/1 [00:00<00:00, 59.72it/s]\n",
      "Epoch 51/1000 - Loss: 7.140234: 100%|██████████| 1/1 [00:00<00:00, 55.43it/s]\n",
      "Epoch 52/1000 - Loss: 7.140377: 100%|██████████| 1/1 [00:00<00:00, 52.10it/s]\n",
      "Epoch 53/1000 - Loss: 7.128059: 100%|██████████| 1/1 [00:00<00:00, 58.34it/s]\n",
      "Epoch 54/1000 - Loss: 7.134853: 100%|██████████| 1/1 [00:00<00:00, 59.32it/s]\n",
      "Epoch 55/1000 - Loss: 7.129679: 100%|██████████| 1/1 [00:00<00:00, 56.05it/s]\n",
      "Epoch 56/1000 - Loss: 7.125426: 100%|██████████| 1/1 [00:00<00:00, 58.46it/s]\n",
      "Epoch 57/1000 - Loss: 7.141139: 100%|██████████| 1/1 [00:00<00:00, 59.17it/s]\n",
      "Epoch 58/1000 - Loss: 6.676735: 100%|██████████| 1/1 [00:00<00:00, 61.54it/s]\n",
      "Epoch 59/1000 - Loss: 7.139787: 100%|██████████| 1/1 [00:00<00:00, 56.17it/s]\n",
      "Epoch 60/1000 - Loss: 7.136817: 100%|██████████| 1/1 [00:00<00:00, 56.60it/s]\n",
      "Epoch 61/1000 - Loss: 7.133941: 100%|██████████| 1/1 [00:00<00:00, 58.28it/s]\n",
      "Epoch 62/1000 - Loss: 7.137100: 100%|██████████| 1/1 [00:00<00:00, 55.35it/s]\n",
      "Epoch 63/1000 - Loss: 7.134811: 100%|██████████| 1/1 [00:00<00:00, 60.16it/s]\n",
      "Epoch 64/1000 - Loss: 7.131270: 100%|██████████| 1/1 [00:00<00:00, 57.52it/s]\n",
      "Epoch 65/1000 - Loss: 7.128592: 100%|██████████| 1/1 [00:00<00:00, 61.38it/s]\n",
      "Epoch 66/1000 - Loss: 7.138403: 100%|██████████| 1/1 [00:00<00:00, 59.65it/s]\n",
      "Epoch 67/1000 - Loss: 7.125481: 100%|██████████| 1/1 [00:00<00:00, 54.04it/s]\n",
      "Epoch 68/1000 - Loss: 7.136485: 100%|██████████| 1/1 [00:00<00:00, 58.25it/s]\n",
      "Epoch 69/1000 - Loss: 7.134478: 100%|██████████| 1/1 [00:00<00:00, 55.27it/s]\n",
      "Epoch 70/1000 - Loss: 7.137722: 100%|██████████| 1/1 [00:00<00:00, 60.55it/s]\n",
      "Epoch 71/1000 - Loss: 7.139524: 100%|██████████| 1/1 [00:00<00:00, 60.45it/s]\n",
      "Epoch 72/1000 - Loss: 7.141446: 100%|██████████| 1/1 [00:00<00:00, 59.09it/s]\n",
      "Epoch 73/1000 - Loss: 7.133758: 100%|██████████| 1/1 [00:00<00:00, 56.66it/s]\n",
      "Epoch 74/1000 - Loss: 7.133408: 100%|██████████| 1/1 [00:00<00:00, 55.46it/s]\n",
      "Epoch 75/1000 - Loss: 7.138188: 100%|██████████| 1/1 [00:00<00:00, 50.64it/s]\n",
      "Epoch 76/1000 - Loss: 7.128291: 100%|██████████| 1/1 [00:00<00:00, 62.84it/s]\n",
      "Epoch 77/1000 - Loss: 7.136433: 100%|██████████| 1/1 [00:00<00:00, 61.11it/s]\n",
      "Epoch 78/1000 - Loss: 7.133207: 100%|██████████| 1/1 [00:00<00:00, 75.60it/s]\n",
      "Epoch 79/1000 - Loss: 7.130334: 100%|██████████| 1/1 [00:00<00:00, 78.91it/s]\n",
      "Epoch 80/1000 - Loss: 7.132847: 100%|██████████| 1/1 [00:00<00:00, 73.98it/s]\n",
      "Epoch 81/1000 - Loss: 6.669794: 100%|██████████| 1/1 [00:00<00:00, 79.35it/s]\n",
      "Epoch 82/1000 - Loss: 7.130432: 100%|██████████| 1/1 [00:00<00:00, 71.30it/s]\n",
      "Epoch 83/1000 - Loss: 7.132143: 100%|██████████| 1/1 [00:00<00:00, 65.30it/s]\n",
      "Epoch 84/1000 - Loss: 7.132525: 100%|██████████| 1/1 [00:00<00:00, 60.76it/s]\n",
      "Epoch 85/1000 - Loss: 7.132768: 100%|██████████| 1/1 [00:00<00:00, 61.58it/s]\n",
      "Epoch 86/1000 - Loss: 7.133246: 100%|██████████| 1/1 [00:00<00:00, 58.77it/s]\n",
      "Epoch 87/1000 - Loss: 7.133335: 100%|██████████| 1/1 [00:00<00:00, 57.79it/s]\n",
      "Epoch 88/1000 - Loss: 7.134753: 100%|██████████| 1/1 [00:00<00:00, 57.11it/s]\n",
      "Epoch 89/1000 - Loss: 6.677737: 100%|██████████| 1/1 [00:00<00:00, 66.55it/s]\n",
      "Epoch 90/1000 - Loss: 7.135230: 100%|██████████| 1/1 [00:00<00:00, 66.51it/s]\n",
      "Epoch 91/1000 - Loss: 7.131320: 100%|██████████| 1/1 [00:00<00:00, 63.64it/s]\n",
      "Epoch 92/1000 - Loss: 7.125553: 100%|██████████| 1/1 [00:00<00:00, 66.14it/s]\n",
      "Epoch 93/1000 - Loss: 7.133430: 100%|██████████| 1/1 [00:00<00:00, 60.23it/s]\n",
      "Epoch 94/1000 - Loss: 7.127140: 100%|██████████| 1/1 [00:00<00:00, 53.13it/s]\n",
      "Epoch 95/1000 - Loss: 7.129335: 100%|██████████| 1/1 [00:00<00:00, 58.78it/s]\n",
      "Epoch 96/1000 - Loss: 7.133941: 100%|██████████| 1/1 [00:00<00:00, 59.25it/s]\n",
      "Epoch 97/1000 - Loss: 7.132652: 100%|██████████| 1/1 [00:00<00:00, 62.62it/s]\n",
      "Epoch 98/1000 - Loss: 7.129957: 100%|██████████| 1/1 [00:00<00:00, 65.21it/s]\n",
      "Epoch 99/1000 - Loss: 7.128006: 100%|██████████| 1/1 [00:00<00:00, 52.14it/s]\n",
      "Epoch 100/1000 - Loss: 7.133110: 100%|██████████| 1/1 [00:00<00:00, 59.57it/s]\n",
      "Epoch 101/1000 - Loss: 7.131766: 100%|██████████| 1/1 [00:00<00:00, 62.42it/s]\n",
      "Epoch 102/1000 - Loss: 7.132406: 100%|██████████| 1/1 [00:00<00:00, 60.57it/s]\n",
      "Epoch 103/1000 - Loss: 7.129493: 100%|██████████| 1/1 [00:00<00:00, 61.12it/s]\n",
      "Epoch 104/1000 - Loss: 7.132756: 100%|██████████| 1/1 [00:00<00:00, 54.97it/s]\n",
      "Epoch 105/1000 - Loss: 7.129937: 100%|██████████| 1/1 [00:00<00:00, 62.43it/s]\n",
      "Epoch 106/1000 - Loss: 7.127761: 100%|██████████| 1/1 [00:00<00:00, 57.38it/s]\n",
      "Epoch 107/1000 - Loss: 7.128659: 100%|██████████| 1/1 [00:00<00:00, 57.78it/s]\n",
      "Epoch 108/1000 - Loss: 7.130981: 100%|██████████| 1/1 [00:00<00:00, 57.93it/s]\n",
      "Epoch 109/1000 - Loss: 7.132267: 100%|██████████| 1/1 [00:00<00:00, 62.18it/s]\n",
      "Epoch 110/1000 - Loss: 7.128358: 100%|██████████| 1/1 [00:00<00:00, 60.92it/s]\n",
      "Epoch 111/1000 - Loss: 7.128937: 100%|██████████| 1/1 [00:00<00:00, 59.67it/s]\n",
      "Epoch 112/1000 - Loss: 7.129117: 100%|██████████| 1/1 [00:00<00:00, 65.57it/s]\n",
      "Epoch 113/1000 - Loss: 7.125367: 100%|██████████| 1/1 [00:00<00:00, 59.29it/s]\n",
      "Epoch 114/1000 - Loss: 7.127223: 100%|██████████| 1/1 [00:00<00:00, 54.80it/s]\n",
      "Epoch 115/1000 - Loss: 7.131788: 100%|██████████| 1/1 [00:00<00:00, 63.60it/s]\n",
      "Epoch 116/1000 - Loss: 7.129494: 100%|██████████| 1/1 [00:00<00:00, 62.17it/s]\n",
      "Epoch 117/1000 - Loss: 7.175915: 100%|██████████| 1/1 [00:00<00:00, 54.14it/s]\n",
      "Epoch 118/1000 - Loss: 7.128078: 100%|██████████| 1/1 [00:00<00:00, 57.77it/s]\n",
      "Epoch 119/1000 - Loss: 7.135187: 100%|██████████| 1/1 [00:00<00:00, 54.64it/s]\n",
      "Epoch 120/1000 - Loss: 7.123005: 100%|██████████| 1/1 [00:00<00:00, 60.90it/s]\n",
      "Epoch 121/1000 - Loss: 7.135582: 100%|██████████| 1/1 [00:00<00:00, 58.53it/s]\n",
      "Epoch 122/1000 - Loss: 7.678477: 100%|██████████| 1/1 [00:00<00:00, 62.70it/s]\n",
      "Epoch 123/1000 - Loss: 7.141773: 100%|██████████| 1/1 [00:00<00:00, 63.51it/s]\n",
      "Epoch 124/1000 - Loss: 7.169194: 100%|██████████| 1/1 [00:00<00:00, 62.65it/s]\n",
      "Epoch 125/1000 - Loss: 7.190949: 100%|██████████| 1/1 [00:00<00:00, 63.48it/s]\n",
      "Epoch 126/1000 - Loss: 7.240039: 100%|██████████| 1/1 [00:00<00:00, 59.78it/s]\n",
      "Epoch 127/1000 - Loss: 7.196466: 100%|██████████| 1/1 [00:00<00:00, 59.05it/s]\n",
      "Epoch 128/1000 - Loss: 7.425453: 100%|██████████| 1/1 [00:00<00:00, 57.66it/s]\n",
      "Epoch 129/1000 - Loss: 7.096653: 100%|██████████| 1/1 [00:00<00:00, 61.53it/s]\n",
      "Epoch 130/1000 - Loss: 7.162322: 100%|██████████| 1/1 [00:00<00:00, 57.49it/s]\n",
      "Epoch 131/1000 - Loss: 7.110505: 100%|██████████| 1/1 [00:00<00:00, 60.53it/s]\n",
      "Epoch 132/1000 - Loss: 7.207773: 100%|██████████| 1/1 [00:00<00:00, 57.11it/s]\n",
      "Epoch 133/1000 - Loss: 7.065339: 100%|██████████| 1/1 [00:00<00:00, 58.68it/s]\n",
      "Epoch 134/1000 - Loss: 7.073013: 100%|██████████| 1/1 [00:00<00:00, 59.08it/s]\n",
      "Epoch 135/1000 - Loss: 6.995471: 100%|██████████| 1/1 [00:00<00:00, 59.40it/s]\n",
      "Epoch 136/1000 - Loss: 7.016880: 100%|██████████| 1/1 [00:00<00:00, 59.97it/s]\n",
      "Epoch 137/1000 - Loss: 6.982705: 100%|██████████| 1/1 [00:00<00:00, 55.43it/s]\n",
      "Epoch 138/1000 - Loss: 6.985016: 100%|██████████| 1/1 [00:00<00:00, 60.77it/s]\n",
      "Epoch 139/1000 - Loss: 6.961644: 100%|██████████| 1/1 [00:00<00:00, 58.74it/s]\n",
      "Epoch 140/1000 - Loss: 6.951331: 100%|██████████| 1/1 [00:00<00:00, 58.42it/s]\n",
      "Epoch 141/1000 - Loss: 6.949302: 100%|██████████| 1/1 [00:00<00:00, 59.42it/s]\n",
      "Epoch 142/1000 - Loss: 6.939589: 100%|██████████| 1/1 [00:00<00:00, 60.58it/s]\n",
      "Epoch 143/1000 - Loss: 6.936991: 100%|██████████| 1/1 [00:00<00:00, 60.99it/s]\n",
      "Epoch 144/1000 - Loss: 6.928984: 100%|██████████| 1/1 [00:00<00:00, 58.96it/s]\n",
      "Epoch 145/1000 - Loss: 6.928730: 100%|██████████| 1/1 [00:00<00:00, 68.52it/s]\n",
      "Epoch 146/1000 - Loss: 6.949193: 100%|██████████| 1/1 [00:00<00:00, 57.79it/s]\n",
      "Epoch 147/1000 - Loss: 6.947566: 100%|██████████| 1/1 [00:00<00:00, 59.33it/s]\n",
      "Epoch 148/1000 - Loss: 6.925373: 100%|██████████| 1/1 [00:00<00:00, 60.22it/s]\n",
      "Epoch 149/1000 - Loss: 6.912599: 100%|██████████| 1/1 [00:00<00:00, 62.97it/s]\n",
      "Epoch 150/1000 - Loss: 6.920910: 100%|██████████| 1/1 [00:00<00:00, 60.47it/s]\n",
      "Epoch 151/1000 - Loss: 6.931154: 100%|██████████| 1/1 [00:00<00:00, 59.36it/s]\n",
      "Epoch 152/1000 - Loss: 6.915920: 100%|██████████| 1/1 [00:00<00:00, 59.27it/s]\n",
      "Epoch 153/1000 - Loss: 6.919826: 100%|██████████| 1/1 [00:00<00:00, 61.98it/s]\n",
      "Epoch 154/1000 - Loss: 6.916511: 100%|██████████| 1/1 [00:00<00:00, 62.09it/s]\n",
      "Epoch 155/1000 - Loss: 6.915287: 100%|██████████| 1/1 [00:00<00:00, 59.21it/s]\n",
      "Epoch 156/1000 - Loss: 6.913543: 100%|██████████| 1/1 [00:00<00:00, 58.62it/s]\n",
      "Epoch 157/1000 - Loss: 6.914195: 100%|██████████| 1/1 [00:00<00:00, 61.02it/s]\n",
      "Epoch 158/1000 - Loss: 6.453608: 100%|██████████| 1/1 [00:00<00:00, 58.96it/s]\n",
      "Epoch 159/1000 - Loss: 6.913430: 100%|██████████| 1/1 [00:00<00:00, 57.67it/s]\n",
      "Epoch 160/1000 - Loss: 6.910076: 100%|██████████| 1/1 [00:00<00:00, 62.56it/s]\n",
      "Epoch 161/1000 - Loss: 6.910670: 100%|██████████| 1/1 [00:00<00:00, 59.00it/s]\n",
      "Epoch 162/1000 - Loss: 6.911370: 100%|██████████| 1/1 [00:00<00:00, 61.20it/s]\n",
      "Epoch 163/1000 - Loss: 6.907299: 100%|██████████| 1/1 [00:00<00:00, 61.20it/s]\n",
      "Epoch 164/1000 - Loss: 6.910888: 100%|██████████| 1/1 [00:00<00:00, 62.30it/s]\n",
      "Epoch 165/1000 - Loss: 6.909083: 100%|██████████| 1/1 [00:00<00:00, 61.37it/s]\n",
      "Epoch 166/1000 - Loss: 6.906570: 100%|██████████| 1/1 [00:00<00:00, 61.35it/s]\n",
      "Epoch 167/1000 - Loss: 6.908254: 100%|██████████| 1/1 [00:00<00:00, 58.99it/s]\n",
      "Epoch 168/1000 - Loss: 6.907158: 100%|██████████| 1/1 [00:00<00:00, 62.19it/s]\n",
      "Epoch 169/1000 - Loss: 6.904543: 100%|██████████| 1/1 [00:00<00:00, 60.87it/s]\n",
      "Epoch 170/1000 - Loss: 6.907681: 100%|██████████| 1/1 [00:00<00:00, 59.43it/s]\n",
      "Epoch 171/1000 - Loss: 6.905321: 100%|██████████| 1/1 [00:00<00:00, 54.06it/s]\n",
      "Epoch 172/1000 - Loss: 6.902454: 100%|██████████| 1/1 [00:00<00:00, 61.95it/s]\n",
      "Epoch 173/1000 - Loss: 6.904426: 100%|██████████| 1/1 [00:00<00:00, 55.17it/s]\n",
      "Epoch 174/1000 - Loss: 6.902486: 100%|██████████| 1/1 [00:00<00:00, 61.12it/s]\n",
      "Epoch 175/1000 - Loss: 6.899085: 100%|██████████| 1/1 [00:00<00:00, 62.45it/s]\n",
      "Epoch 176/1000 - Loss: 6.903934: 100%|██████████| 1/1 [00:00<00:00, 63.62it/s]\n",
      "Epoch 177/1000 - Loss: 6.904041: 100%|██████████| 1/1 [00:00<00:00, 57.63it/s]\n",
      "Epoch 178/1000 - Loss: 6.903560: 100%|██████████| 1/1 [00:00<00:00, 56.30it/s]\n",
      "Epoch 179/1000 - Loss: 6.900638: 100%|██████████| 1/1 [00:00<00:00, 62.06it/s]\n",
      "Epoch 180/1000 - Loss: 6.904865: 100%|██████████| 1/1 [00:00<00:00, 60.22it/s]\n",
      "Epoch 181/1000 - Loss: 6.900853: 100%|██████████| 1/1 [00:00<00:00, 56.21it/s]\n",
      "Epoch 182/1000 - Loss: 6.905643: 100%|██████████| 1/1 [00:00<00:00, 53.31it/s]\n",
      "Epoch 183/1000 - Loss: 6.904786: 100%|██████████| 1/1 [00:00<00:00, 58.81it/s]\n",
      "Epoch 184/1000 - Loss: 6.901428: 100%|██████████| 1/1 [00:00<00:00, 61.58it/s]\n",
      "Epoch 185/1000 - Loss: 6.900918: 100%|██████████| 1/1 [00:00<00:00, 57.38it/s]\n",
      "Epoch 186/1000 - Loss: 6.901515: 100%|██████████| 1/1 [00:00<00:00, 60.38it/s]\n",
      "Epoch 187/1000 - Loss: 6.897972: 100%|██████████| 1/1 [00:00<00:00, 60.77it/s]\n",
      "Epoch 188/1000 - Loss: 6.902725: 100%|██████████| 1/1 [00:00<00:00, 58.70it/s]\n",
      "Epoch 189/1000 - Loss: 6.903022: 100%|██████████| 1/1 [00:00<00:00, 58.67it/s]\n",
      "Epoch 190/1000 - Loss: 6.901037: 100%|██████████| 1/1 [00:00<00:00, 55.49it/s]\n",
      "Epoch 191/1000 - Loss: 6.899247: 100%|██████████| 1/1 [00:00<00:00, 57.58it/s]\n",
      "Epoch 192/1000 - Loss: 6.896516: 100%|██████████| 1/1 [00:00<00:00, 60.03it/s]\n",
      "Epoch 193/1000 - Loss: 6.900295: 100%|██████████| 1/1 [00:00<00:00, 58.71it/s]\n",
      "Epoch 194/1000 - Loss: 6.899930: 100%|██████████| 1/1 [00:00<00:00, 60.19it/s]\n",
      "Epoch 195/1000 - Loss: 6.901254: 100%|██████████| 1/1 [00:00<00:00, 59.13it/s]\n",
      "Epoch 196/1000 - Loss: 6.899088: 100%|██████████| 1/1 [00:00<00:00, 61.68it/s]\n",
      "Epoch 197/1000 - Loss: 6.898805: 100%|██████████| 1/1 [00:00<00:00, 62.66it/s]\n",
      "Epoch 198/1000 - Loss: 6.898341: 100%|██████████| 1/1 [00:00<00:00, 58.56it/s]\n",
      "Epoch 199/1000 - Loss: 6.897909: 100%|██████████| 1/1 [00:00<00:00, 58.61it/s]\n",
      "Epoch 200/1000 - Loss: 6.896026: 100%|██████████| 1/1 [00:00<00:00, 57.30it/s]\n",
      "Epoch 201/1000 - Loss: 6.895376: 100%|██████████| 1/1 [00:00<00:00, 57.50it/s]\n",
      "Epoch 202/1000 - Loss: 6.897655: 100%|██████████| 1/1 [00:00<00:00, 57.02it/s]\n",
      "Epoch 203/1000 - Loss: 6.898490: 100%|██████████| 1/1 [00:00<00:00, 59.14it/s]\n",
      "Epoch 204/1000 - Loss: 6.893129: 100%|██████████| 1/1 [00:00<00:00, 64.78it/s]\n",
      "Epoch 205/1000 - Loss: 6.895116: 100%|██████████| 1/1 [00:00<00:00, 55.71it/s]\n",
      "Epoch 206/1000 - Loss: 6.895838: 100%|██████████| 1/1 [00:00<00:00, 60.50it/s]\n",
      "Epoch 207/1000 - Loss: 6.891996: 100%|██████████| 1/1 [00:00<00:00, 48.18it/s]\n",
      "Epoch 208/1000 - Loss: 6.441633: 100%|██████████| 1/1 [00:00<00:00, 58.63it/s]\n",
      "Epoch 209/1000 - Loss: 6.894658: 100%|██████████| 1/1 [00:00<00:00, 60.69it/s]\n",
      "Epoch 210/1000 - Loss: 6.894489: 100%|██████████| 1/1 [00:00<00:00, 60.40it/s]\n",
      "Epoch 211/1000 - Loss: 6.892934: 100%|██████████| 1/1 [00:00<00:00, 63.13it/s]\n",
      "Epoch 212/1000 - Loss: 6.892396: 100%|██████████| 1/1 [00:00<00:00, 61.95it/s]\n",
      "Epoch 213/1000 - Loss: 6.892004: 100%|██████████| 1/1 [00:00<00:00, 61.15it/s]\n",
      "Epoch 214/1000 - Loss: 6.890976: 100%|██████████| 1/1 [00:00<00:00, 58.69it/s]\n",
      "Epoch 215/1000 - Loss: 6.894061: 100%|██████████| 1/1 [00:00<00:00, 59.95it/s]\n",
      "Epoch 216/1000 - Loss: 6.891223: 100%|██████████| 1/1 [00:00<00:00, 63.86it/s]\n",
      "Epoch 217/1000 - Loss: 6.889431: 100%|██████████| 1/1 [00:00<00:00, 66.66it/s]\n",
      "Epoch 218/1000 - Loss: 6.886852: 100%|██████████| 1/1 [00:00<00:00, 64.65it/s]\n",
      "Epoch 219/1000 - Loss: 6.889596: 100%|██████████| 1/1 [00:00<00:00, 63.63it/s]\n",
      "Epoch 220/1000 - Loss: 6.887395: 100%|██████████| 1/1 [00:00<00:00, 66.29it/s]\n",
      "Epoch 221/1000 - Loss: 6.889111: 100%|██████████| 1/1 [00:00<00:00, 62.97it/s]\n",
      "Epoch 222/1000 - Loss: 6.886479: 100%|██████████| 1/1 [00:00<00:00, 58.55it/s]\n",
      "Epoch 223/1000 - Loss: 6.885377: 100%|██████████| 1/1 [00:00<00:00, 60.09it/s]\n",
      "Epoch 224/1000 - Loss: 6.887935: 100%|██████████| 1/1 [00:00<00:00, 57.59it/s]\n",
      "Epoch 225/1000 - Loss: 6.887551: 100%|██████████| 1/1 [00:00<00:00, 57.39it/s]\n",
      "Epoch 226/1000 - Loss: 6.886948: 100%|██████████| 1/1 [00:00<00:00, 60.00it/s]\n",
      "Epoch 227/1000 - Loss: 6.883632: 100%|██████████| 1/1 [00:00<00:00, 57.97it/s]\n",
      "Epoch 228/1000 - Loss: 6.885036: 100%|██████████| 1/1 [00:00<00:00, 60.29it/s]\n",
      "Epoch 229/1000 - Loss: 6.883340: 100%|██████████| 1/1 [00:00<00:00, 62.24it/s]\n",
      "Epoch 230/1000 - Loss: 6.878799: 100%|██████████| 1/1 [00:00<00:00, 60.41it/s]\n",
      "Epoch 231/1000 - Loss: 6.883364: 100%|██████████| 1/1 [00:00<00:00, 60.26it/s]\n",
      "Epoch 232/1000 - Loss: 6.881562: 100%|██████████| 1/1 [00:00<00:00, 59.59it/s]\n",
      "Epoch 233/1000 - Loss: 6.882691: 100%|██████████| 1/1 [00:00<00:00, 59.19it/s]\n",
      "Epoch 234/1000 - Loss: 6.882025: 100%|██████████| 1/1 [00:00<00:00, 58.84it/s]\n",
      "Epoch 235/1000 - Loss: 6.882010: 100%|██████████| 1/1 [00:00<00:00, 61.20it/s]\n",
      "Epoch 236/1000 - Loss: 6.882437: 100%|██████████| 1/1 [00:00<00:00, 60.85it/s]\n",
      "Epoch 237/1000 - Loss: 6.879045: 100%|██████████| 1/1 [00:00<00:00, 61.43it/s]\n",
      "Epoch 238/1000 - Loss: 6.878680: 100%|██████████| 1/1 [00:00<00:00, 65.43it/s]\n",
      "Epoch 239/1000 - Loss: 6.875690: 100%|██████████| 1/1 [00:00<00:00, 58.48it/s]\n",
      "Epoch 240/1000 - Loss: 6.873753: 100%|██████████| 1/1 [00:00<00:00, 50.53it/s]\n",
      "Epoch 241/1000 - Loss: 6.883852: 100%|██████████| 1/1 [00:00<00:00, 60.77it/s]\n",
      "Epoch 242/1000 - Loss: 7.169313: 100%|██████████| 1/1 [00:00<00:00, 63.84it/s]\n",
      "Epoch 243/1000 - Loss: 6.874564: 100%|██████████| 1/1 [00:00<00:00, 69.17it/s]\n",
      "Epoch 244/1000 - Loss: 6.878030: 100%|██████████| 1/1 [00:00<00:00, 69.24it/s]\n",
      "Epoch 245/1000 - Loss: 6.873906: 100%|██████████| 1/1 [00:00<00:00, 63.21it/s]\n",
      "Epoch 246/1000 - Loss: 6.874217: 100%|██████████| 1/1 [00:00<00:00, 61.61it/s]\n",
      "Epoch 247/1000 - Loss: 6.872511: 100%|██████████| 1/1 [00:00<00:00, 62.33it/s]\n",
      "Epoch 248/1000 - Loss: 6.870635: 100%|██████████| 1/1 [00:00<00:00, 60.87it/s]\n",
      "Epoch 249/1000 - Loss: 6.871564: 100%|██████████| 1/1 [00:00<00:00, 62.79it/s]\n",
      "Epoch 250/1000 - Loss: 6.865595: 100%|██████████| 1/1 [00:00<00:00, 59.99it/s]\n",
      "Epoch 251/1000 - Loss: 6.863980: 100%|██████████| 1/1 [00:00<00:00, 60.73it/s]\n",
      "Epoch 252/1000 - Loss: 6.867926: 100%|██████████| 1/1 [00:00<00:00, 61.03it/s]\n",
      "Epoch 253/1000 - Loss: 6.868049: 100%|██████████| 1/1 [00:00<00:00, 59.45it/s]\n",
      "Epoch 254/1000 - Loss: 6.863174: 100%|██████████| 1/1 [00:00<00:00, 60.12it/s]\n",
      "Epoch 255/1000 - Loss: 6.862421: 100%|██████████| 1/1 [00:00<00:00, 58.97it/s]\n",
      "Epoch 256/1000 - Loss: 6.862293: 100%|██████████| 1/1 [00:00<00:00, 60.50it/s]\n",
      "Epoch 257/1000 - Loss: 6.859009: 100%|██████████| 1/1 [00:00<00:00, 68.62it/s]\n",
      "Epoch 258/1000 - Loss: 6.857159: 100%|██████████| 1/1 [00:00<00:00, 63.07it/s]\n",
      "Epoch 259/1000 - Loss: 6.853368: 100%|██████████| 1/1 [00:00<00:00, 64.23it/s]\n",
      "Epoch 260/1000 - Loss: 6.851813: 100%|██████████| 1/1 [00:00<00:00, 64.88it/s]\n",
      "Epoch 261/1000 - Loss: 6.849119: 100%|██████████| 1/1 [00:00<00:00, 70.41it/s]\n",
      "Epoch 262/1000 - Loss: 6.847171: 100%|██████████| 1/1 [00:00<00:00, 66.65it/s]\n",
      "Epoch 263/1000 - Loss: 6.842633: 100%|██████████| 1/1 [00:00<00:00, 68.42it/s]\n",
      "Epoch 264/1000 - Loss: 6.834975: 100%|██████████| 1/1 [00:00<00:00, 45.48it/s]\n",
      "Epoch 265/1000 - Loss: 6.826189: 100%|██████████| 1/1 [00:00<00:00, 61.96it/s]\n",
      "Epoch 266/1000 - Loss: 6.815726: 100%|██████████| 1/1 [00:00<00:00, 62.26it/s]\n",
      "Epoch 267/1000 - Loss: 6.766670: 100%|██████████| 1/1 [00:00<00:00, 61.43it/s]\n",
      "Epoch 268/1000 - Loss: 6.739254: 100%|██████████| 1/1 [00:00<00:00, 64.58it/s]\n",
      "Epoch 269/1000 - Loss: 6.679025: 100%|██████████| 1/1 [00:00<00:00, 66.38it/s]\n",
      "Epoch 270/1000 - Loss: 6.621036: 100%|██████████| 1/1 [00:00<00:00, 65.02it/s]\n",
      "Epoch 271/1000 - Loss: 6.521749: 100%|██████████| 1/1 [00:00<00:00, 62.42it/s]\n",
      "Epoch 272/1000 - Loss: 6.450549: 100%|██████████| 1/1 [00:00<00:00, 60.10it/s]\n",
      "Epoch 273/1000 - Loss: 6.482510: 100%|██████████| 1/1 [00:00<00:00, 60.17it/s]\n",
      "Epoch 274/1000 - Loss: 6.481028: 100%|██████████| 1/1 [00:00<00:00, 54.86it/s]\n",
      "Epoch 275/1000 - Loss: 6.504412: 100%|██████████| 1/1 [00:00<00:00, 66.85it/s]\n",
      "Epoch 276/1000 - Loss: 6.447289: 100%|██████████| 1/1 [00:00<00:00, 61.74it/s]\n",
      "Epoch 277/1000 - Loss: 6.459333: 100%|██████████| 1/1 [00:00<00:00, 62.53it/s]\n",
      "Epoch 278/1000 - Loss: 6.440762: 100%|██████████| 1/1 [00:00<00:00, 64.74it/s]\n",
      "Epoch 279/1000 - Loss: 6.426890: 100%|██████████| 1/1 [00:00<00:00, 64.71it/s]\n",
      "Epoch 280/1000 - Loss: 6.439804: 100%|██████████| 1/1 [00:00<00:00, 63.83it/s]\n",
      "Epoch 281/1000 - Loss: 6.432795: 100%|██████████| 1/1 [00:00<00:00, 63.45it/s]\n",
      "Epoch 282/1000 - Loss: 6.435116: 100%|██████████| 1/1 [00:00<00:00, 62.02it/s]\n",
      "Epoch 283/1000 - Loss: 6.428900: 100%|██████████| 1/1 [00:00<00:00, 63.44it/s]\n",
      "Epoch 284/1000 - Loss: 6.430254: 100%|██████████| 1/1 [00:00<00:00, 64.35it/s]\n",
      "Epoch 285/1000 - Loss: 6.433166: 100%|██████████| 1/1 [00:00<00:00, 65.06it/s]\n",
      "Epoch 286/1000 - Loss: 6.431995: 100%|██████████| 1/1 [00:00<00:00, 61.83it/s]\n",
      "Epoch 287/1000 - Loss: 6.416249: 100%|██████████| 1/1 [00:00<00:00, 61.41it/s]\n",
      "Epoch 288/1000 - Loss: 6.411883: 100%|██████████| 1/1 [00:00<00:00, 63.08it/s]\n",
      "Epoch 289/1000 - Loss: 6.412133: 100%|██████████| 1/1 [00:00<00:00, 65.42it/s]\n",
      "Epoch 290/1000 - Loss: 6.014569: 100%|██████████| 1/1 [00:00<00:00, 61.67it/s]\n",
      "Epoch 291/1000 - Loss: 6.387625: 100%|██████████| 1/1 [00:00<00:00, 61.15it/s]\n",
      "Epoch 292/1000 - Loss: 6.400787: 100%|██████████| 1/1 [00:00<00:00, 60.56it/s]\n",
      "Epoch 293/1000 - Loss: 6.368015: 100%|██████████| 1/1 [00:00<00:00, 63.62it/s]\n",
      "Epoch 294/1000 - Loss: 6.320752: 100%|██████████| 1/1 [00:00<00:00, 66.45it/s]\n",
      "Epoch 295/1000 - Loss: 6.230957: 100%|██████████| 1/1 [00:00<00:00, 63.87it/s]\n",
      "Epoch 296/1000 - Loss: 6.101792: 100%|██████████| 1/1 [00:00<00:00, 63.69it/s]\n",
      "Epoch 297/1000 - Loss: 6.115165: 100%|██████████| 1/1 [00:00<00:00, 62.18it/s]\n",
      "Epoch 298/1000 - Loss: 6.022470: 100%|██████████| 1/1 [00:00<00:00, 63.37it/s]\n",
      "Epoch 299/1000 - Loss: 6.002018: 100%|██████████| 1/1 [00:00<00:00, 65.01it/s]\n",
      "Epoch 300/1000 - Loss: 5.995802: 100%|██████████| 1/1 [00:00<00:00, 63.71it/s]\n",
      "Epoch 301/1000 - Loss: 5.989807: 100%|██████████| 1/1 [00:00<00:00, 58.99it/s]\n",
      "Epoch 302/1000 - Loss: 5.995080: 100%|██████████| 1/1 [00:00<00:00, 63.07it/s]\n",
      "Epoch 303/1000 - Loss: 5.971343: 100%|██████████| 1/1 [00:00<00:00, 61.85it/s]\n",
      "Epoch 304/1000 - Loss: 5.978868: 100%|██████████| 1/1 [00:00<00:00, 63.53it/s]\n",
      "Epoch 305/1000 - Loss: 5.973645: 100%|██████████| 1/1 [00:00<00:00, 61.61it/s]\n",
      "Epoch 306/1000 - Loss: 5.973846: 100%|██████████| 1/1 [00:00<00:00, 52.10it/s]\n",
      "Epoch 307/1000 - Loss: 5.962541: 100%|██████████| 1/1 [00:00<00:00, 62.62it/s]\n",
      "Epoch 308/1000 - Loss: 5.963686: 100%|██████████| 1/1 [00:00<00:00, 61.73it/s]\n",
      "Epoch 309/1000 - Loss: 5.958862: 100%|██████████| 1/1 [00:00<00:00, 59.80it/s]\n",
      "Epoch 310/1000 - Loss: 5.957481: 100%|██████████| 1/1 [00:00<00:00, 59.44it/s]\n",
      "Epoch 311/1000 - Loss: 5.954315: 100%|██████████| 1/1 [00:00<00:00, 60.76it/s]\n",
      "Epoch 312/1000 - Loss: 5.954854: 100%|██████████| 1/1 [00:00<00:00, 55.72it/s]\n",
      "Epoch 313/1000 - Loss: 5.951588: 100%|██████████| 1/1 [00:00<00:00, 61.91it/s]\n",
      "Epoch 314/1000 - Loss: 5.950618: 100%|██████████| 1/1 [00:00<00:00, 61.10it/s]\n",
      "Epoch 315/1000 - Loss: 5.949605: 100%|██████████| 1/1 [00:00<00:00, 60.67it/s]\n",
      "Epoch 316/1000 - Loss: 5.950559: 100%|██████████| 1/1 [00:00<00:00, 63.23it/s]\n",
      "Epoch 317/1000 - Loss: 5.947476: 100%|██████████| 1/1 [00:00<00:00, 68.82it/s]\n",
      "Epoch 318/1000 - Loss: 5.948453: 100%|██████████| 1/1 [00:00<00:00, 61.54it/s]\n",
      "Epoch 319/1000 - Loss: 5.947572: 100%|██████████| 1/1 [00:00<00:00, 67.35it/s]\n",
      "Epoch 320/1000 - Loss: 5.947104: 100%|██████████| 1/1 [00:00<00:00, 72.73it/s]\n",
      "Epoch 321/1000 - Loss: 5.944578: 100%|██████████| 1/1 [00:00<00:00, 63.30it/s]\n",
      "Epoch 322/1000 - Loss: 5.945804: 100%|██████████| 1/1 [00:00<00:00, 65.47it/s]\n",
      "Epoch 323/1000 - Loss: 5.941700: 100%|██████████| 1/1 [00:00<00:00, 62.66it/s]\n",
      "Epoch 324/1000 - Loss: 5.946530: 100%|██████████| 1/1 [00:00<00:00, 63.27it/s]\n",
      "Epoch 325/1000 - Loss: 5.945178: 100%|██████████| 1/1 [00:00<00:00, 67.67it/s]\n",
      "Epoch 326/1000 - Loss: 5.946078: 100%|██████████| 1/1 [00:00<00:00, 63.89it/s]\n",
      "Epoch 327/1000 - Loss: 5.945162: 100%|██████████| 1/1 [00:00<00:00, 64.86it/s]\n",
      "Epoch 328/1000 - Loss: 5.945881: 100%|██████████| 1/1 [00:00<00:00, 64.03it/s]\n",
      "Epoch 329/1000 - Loss: 5.945919: 100%|██████████| 1/1 [00:00<00:00, 64.58it/s]\n",
      "Epoch 330/1000 - Loss: 5.943540: 100%|██████████| 1/1 [00:00<00:00, 61.04it/s]\n",
      "Epoch 331/1000 - Loss: 5.944541: 100%|██████████| 1/1 [00:00<00:00, 61.04it/s]\n",
      "Epoch 332/1000 - Loss: 5.944375: 100%|██████████| 1/1 [00:00<00:00, 60.98it/s]\n",
      "Epoch 333/1000 - Loss: 5.943400: 100%|██████████| 1/1 [00:00<00:00, 59.43it/s]\n",
      "Epoch 334/1000 - Loss: 5.943424: 100%|██████████| 1/1 [00:00<00:00, 65.13it/s]\n",
      "Epoch 335/1000 - Loss: 5.945272: 100%|██████████| 1/1 [00:00<00:00, 62.57it/s]\n",
      "Epoch 336/1000 - Loss: 5.945972: 100%|██████████| 1/1 [00:00<00:00, 60.30it/s]\n",
      "Epoch 337/1000 - Loss: 5.944057: 100%|██████████| 1/1 [00:00<00:00, 61.73it/s]\n",
      "Epoch 338/1000 - Loss: 5.944705: 100%|██████████| 1/1 [00:00<00:00, 63.73it/s]\n",
      "Epoch 339/1000 - Loss: 5.942701: 100%|██████████| 1/1 [00:00<00:00, 64.32it/s]\n",
      "Epoch 340/1000 - Loss: 5.943328: 100%|██████████| 1/1 [00:00<00:00, 62.42it/s]\n",
      "Epoch 341/1000 - Loss: 5.942059: 100%|██████████| 1/1 [00:00<00:00, 59.21it/s]\n",
      "Epoch 342/1000 - Loss: 5.943940: 100%|██████████| 1/1 [00:00<00:00, 62.96it/s]\n",
      "Epoch 343/1000 - Loss: 5.943380: 100%|██████████| 1/1 [00:00<00:00, 62.87it/s]\n",
      "Epoch 344/1000 - Loss: 5.944767: 100%|██████████| 1/1 [00:00<00:00, 60.58it/s]\n",
      "Epoch 345/1000 - Loss: 5.943979: 100%|██████████| 1/1 [00:00<00:00, 61.60it/s]\n",
      "Epoch 346/1000 - Loss: 5.944057: 100%|██████████| 1/1 [00:00<00:00, 59.76it/s]\n",
      "Epoch 347/1000 - Loss: 5.945274: 100%|██████████| 1/1 [00:00<00:00, 55.91it/s]\n",
      "Epoch 348/1000 - Loss: 5.943510: 100%|██████████| 1/1 [00:00<00:00, 64.72it/s]\n",
      "Epoch 349/1000 - Loss: 5.943595: 100%|██████████| 1/1 [00:00<00:00, 59.75it/s]\n",
      "Epoch 350/1000 - Loss: 5.942959: 100%|██████████| 1/1 [00:00<00:00, 63.82it/s]\n",
      "Epoch 351/1000 - Loss: 5.943532: 100%|██████████| 1/1 [00:00<00:00, 58.91it/s]\n",
      "Epoch 352/1000 - Loss: 5.943737: 100%|██████████| 1/1 [00:00<00:00, 62.09it/s]\n",
      "Epoch 353/1000 - Loss: 5.942973: 100%|██████████| 1/1 [00:00<00:00, 58.22it/s]\n",
      "Epoch 354/1000 - Loss: 5.940129: 100%|██████████| 1/1 [00:00<00:00, 60.82it/s]\n",
      "Epoch 355/1000 - Loss: 5.942783: 100%|██████████| 1/1 [00:00<00:00, 60.30it/s]\n",
      "Epoch 356/1000 - Loss: 5.943258: 100%|██████████| 1/1 [00:00<00:00, 64.00it/s]\n",
      "Epoch 357/1000 - Loss: 5.943414: 100%|██████████| 1/1 [00:00<00:00, 65.96it/s]\n",
      "Epoch 358/1000 - Loss: 5.942367: 100%|██████████| 1/1 [00:00<00:00, 61.53it/s]\n",
      "Epoch 359/1000 - Loss: 5.943160: 100%|██████████| 1/1 [00:00<00:00, 63.81it/s]\n",
      "Epoch 360/1000 - Loss: 5.942643: 100%|██████████| 1/1 [00:00<00:00, 63.32it/s]\n",
      "Epoch 361/1000 - Loss: 5.943856: 100%|██████████| 1/1 [00:00<00:00, 61.37it/s]\n",
      "Epoch 362/1000 - Loss: 5.941741: 100%|██████████| 1/1 [00:00<00:00, 69.17it/s]\n",
      "Epoch 363/1000 - Loss: 5.943769: 100%|██████████| 1/1 [00:00<00:00, 80.52it/s]\n",
      "Epoch 364/1000 - Loss: 5.943310: 100%|██████████| 1/1 [00:00<00:00, 71.34it/s]\n",
      "Epoch 365/1000 - Loss: 5.943512: 100%|██████████| 1/1 [00:00<00:00, 77.95it/s]\n",
      "Epoch 366/1000 - Loss: 5.943131: 100%|██████████| 1/1 [00:00<00:00, 69.85it/s]\n",
      "Epoch 367/1000 - Loss: 5.942307: 100%|██████████| 1/1 [00:00<00:00, 66.43it/s]\n",
      "Epoch 368/1000 - Loss: 5.942635: 100%|██████████| 1/1 [00:00<00:00, 60.58it/s]\n",
      "Epoch 369/1000 - Loss: 5.942267: 100%|██████████| 1/1 [00:00<00:00, 60.90it/s]\n",
      "Epoch 370/1000 - Loss: 5.943245: 100%|██████████| 1/1 [00:00<00:00, 65.55it/s]\n",
      "Epoch 371/1000 - Loss: 5.942008: 100%|██████████| 1/1 [00:00<00:00, 65.06it/s]\n",
      "Epoch 372/1000 - Loss: 5.942153: 100%|██████████| 1/1 [00:00<00:00, 60.83it/s]\n",
      "Epoch 373/1000 - Loss: 5.942734: 100%|██████████| 1/1 [00:00<00:00, 59.96it/s]\n",
      "Epoch 374/1000 - Loss: 5.943036: 100%|██████████| 1/1 [00:00<00:00, 64.84it/s]\n",
      "Epoch 375/1000 - Loss: 5.942206: 100%|██████████| 1/1 [00:00<00:00, 70.23it/s]\n",
      "Epoch 376/1000 - Loss: 5.942041: 100%|██████████| 1/1 [00:00<00:00, 72.54it/s]\n",
      "Epoch 377/1000 - Loss: 5.941819: 100%|██████████| 1/1 [00:00<00:00, 62.61it/s]\n",
      "Epoch 378/1000 - Loss: 5.942268: 100%|██████████| 1/1 [00:00<00:00, 47.20it/s]\n",
      "Epoch 379/1000 - Loss: 5.940949: 100%|██████████| 1/1 [00:00<00:00, 59.85it/s]\n",
      "Epoch 380/1000 - Loss: 5.941423: 100%|██████████| 1/1 [00:00<00:00, 62.83it/s]\n",
      "Epoch 381/1000 - Loss: 5.940784: 100%|██████████| 1/1 [00:00<00:00, 60.31it/s]\n",
      "Epoch 382/1000 - Loss: 5.942275: 100%|██████████| 1/1 [00:00<00:00, 65.27it/s]\n",
      "Epoch 383/1000 - Loss: 5.942313: 100%|██████████| 1/1 [00:00<00:00, 68.06it/s]\n",
      "Epoch 384/1000 - Loss: 5.942062: 100%|██████████| 1/1 [00:00<00:00, 81.34it/s]\n",
      "Epoch 385/1000 - Loss: 5.941225: 100%|██████████| 1/1 [00:00<00:00, 85.88it/s]\n",
      "Epoch 386/1000 - Loss: 5.941196: 100%|██████████| 1/1 [00:00<00:00, 84.05it/s]\n",
      "Epoch 387/1000 - Loss: 5.941505: 100%|██████████| 1/1 [00:00<00:00, 75.32it/s]\n",
      "Epoch 388/1000 - Loss: 5.941472: 100%|██████████| 1/1 [00:00<00:00, 76.80it/s]\n",
      "Epoch 389/1000 - Loss: 5.941671: 100%|██████████| 1/1 [00:00<00:00, 78.41it/s]\n",
      "Epoch 390/1000 - Loss: 5.941948: 100%|██████████| 1/1 [00:00<00:00, 77.26it/s]\n",
      "Epoch 391/1000 - Loss: 5.942224: 100%|██████████| 1/1 [00:00<00:00, 85.73it/s]\n",
      "Epoch 392/1000 - Loss: 5.941945: 100%|██████████| 1/1 [00:00<00:00, 79.28it/s]\n",
      "Epoch 393/1000 - Loss: 5.941285: 100%|██████████| 1/1 [00:00<00:00, 72.76it/s]\n",
      "Epoch 394/1000 - Loss: 5.940734: 100%|██████████| 1/1 [00:00<00:00, 71.17it/s]\n",
      "Epoch 395/1000 - Loss: 5.939839: 100%|██████████| 1/1 [00:00<00:00, 66.04it/s]\n",
      "Epoch 396/1000 - Loss: 5.940839: 100%|██████████| 1/1 [00:00<00:00, 60.86it/s]\n",
      "Epoch 397/1000 - Loss: 5.940521: 100%|██████████| 1/1 [00:00<00:00, 65.65it/s]\n",
      "Epoch 398/1000 - Loss: 5.941878: 100%|██████████| 1/1 [00:00<00:00, 62.63it/s]\n",
      "Epoch 399/1000 - Loss: 5.940534: 100%|██████████| 1/1 [00:00<00:00, 66.12it/s]\n",
      "Epoch 400/1000 - Loss: 5.940922: 100%|██████████| 1/1 [00:00<00:00, 59.36it/s]\n",
      "Epoch 401/1000 - Loss: 5.941162: 100%|██████████| 1/1 [00:00<00:00, 61.26it/s]\n",
      "Epoch 402/1000 - Loss: 5.940186: 100%|██████████| 1/1 [00:00<00:00, 60.95it/s]\n",
      "Epoch 403/1000 - Loss: 5.940981: 100%|██████████| 1/1 [00:00<00:00, 63.17it/s]\n",
      "Epoch 404/1000 - Loss: 5.940942: 100%|██████████| 1/1 [00:00<00:00, 56.22it/s]\n",
      "Epoch 405/1000 - Loss: 5.940947: 100%|██████████| 1/1 [00:00<00:00, 61.00it/s]\n",
      "Epoch 406/1000 - Loss: 5.940002: 100%|██████████| 1/1 [00:00<00:00, 61.66it/s]\n",
      "Epoch 407/1000 - Loss: 5.940333: 100%|██████████| 1/1 [00:00<00:00, 62.27it/s]\n",
      "Epoch 408/1000 - Loss: 5.940694: 100%|██████████| 1/1 [00:00<00:00, 60.79it/s]\n",
      "Epoch 409/1000 - Loss: 5.940777: 100%|██████████| 1/1 [00:00<00:00, 58.73it/s]\n",
      "Epoch 410/1000 - Loss: 5.940602: 100%|██████████| 1/1 [00:00<00:00, 63.88it/s]\n",
      "Epoch 411/1000 - Loss: 5.940766: 100%|██████████| 1/1 [00:00<00:00, 67.01it/s]\n",
      "Epoch 412/1000 - Loss: 5.940291: 100%|██████████| 1/1 [00:00<00:00, 59.70it/s]\n",
      "Epoch 413/1000 - Loss: 5.940008: 100%|██████████| 1/1 [00:00<00:00, 61.49it/s]\n",
      "Epoch 414/1000 - Loss: 5.939864: 100%|██████████| 1/1 [00:00<00:00, 59.30it/s]\n",
      "Epoch 415/1000 - Loss: 5.939548: 100%|██████████| 1/1 [00:00<00:00, 62.26it/s]\n",
      "Epoch 416/1000 - Loss: 5.940418: 100%|██████████| 1/1 [00:00<00:00, 60.77it/s]\n",
      "Epoch 417/1000 - Loss: 5.940296: 100%|██████████| 1/1 [00:00<00:00, 62.81it/s]\n",
      "Epoch 418/1000 - Loss: 5.938865: 100%|██████████| 1/1 [00:00<00:00, 61.43it/s]\n",
      "Epoch 419/1000 - Loss: 5.940011: 100%|██████████| 1/1 [00:00<00:00, 64.30it/s]\n",
      "Epoch 420/1000 - Loss: 5.940301: 100%|██████████| 1/1 [00:00<00:00, 58.31it/s]\n",
      "Epoch 421/1000 - Loss: 5.940505: 100%|██████████| 1/1 [00:00<00:00, 61.87it/s]\n",
      "Epoch 422/1000 - Loss: 5.939482: 100%|██████████| 1/1 [00:00<00:00, 61.60it/s]\n",
      "Epoch 423/1000 - Loss: 5.940019: 100%|██████████| 1/1 [00:00<00:00, 61.00it/s]\n",
      "Epoch 424/1000 - Loss: 5.938726: 100%|██████████| 1/1 [00:00<00:00, 61.04it/s]\n",
      "Epoch 425/1000 - Loss: 5.939172: 100%|██████████| 1/1 [00:00<00:00, 62.02it/s]\n",
      "Epoch 426/1000 - Loss: 5.937303: 100%|██████████| 1/1 [00:00<00:00, 59.29it/s]\n",
      "Epoch 427/1000 - Loss: 5.937753: 100%|██████████| 1/1 [00:00<00:00, 61.33it/s]\n",
      "Epoch 428/1000 - Loss: 5.938775: 100%|██████████| 1/1 [00:00<00:00, 59.25it/s]\n",
      "Epoch 429/1000 - Loss: 5.939311: 100%|██████████| 1/1 [00:00<00:00, 62.28it/s]\n",
      "Epoch 430/1000 - Loss: 5.938784: 100%|██████████| 1/1 [00:00<00:00, 59.71it/s]\n",
      "Epoch 431/1000 - Loss: 5.938627: 100%|██████████| 1/1 [00:00<00:00, 57.51it/s]\n",
      "Epoch 432/1000 - Loss: 5.939454: 100%|██████████| 1/1 [00:00<00:00, 59.50it/s]\n",
      "Epoch 433/1000 - Loss: 5.939538: 100%|██████████| 1/1 [00:00<00:00, 61.45it/s]\n",
      "Epoch 434/1000 - Loss: 5.939102: 100%|██████████| 1/1 [00:00<00:00, 64.35it/s]\n",
      "Epoch 435/1000 - Loss: 5.938819: 100%|██████████| 1/1 [00:00<00:00, 59.70it/s]\n",
      "Epoch 436/1000 - Loss: 5.938582: 100%|██████████| 1/1 [00:00<00:00, 59.45it/s]\n",
      "Epoch 437/1000 - Loss: 5.938681: 100%|██████████| 1/1 [00:00<00:00, 59.18it/s]\n",
      "Epoch 438/1000 - Loss: 5.938091: 100%|██████████| 1/1 [00:00<00:00, 56.75it/s]\n",
      "Epoch 439/1000 - Loss: 5.939231: 100%|██████████| 1/1 [00:00<00:00, 58.90it/s]\n",
      "Epoch 440/1000 - Loss: 5.938728: 100%|██████████| 1/1 [00:00<00:00, 57.50it/s]\n",
      "Epoch 441/1000 - Loss: 5.938310: 100%|██████████| 1/1 [00:00<00:00, 63.24it/s]\n",
      "Epoch 442/1000 - Loss: 5.938263: 100%|██████████| 1/1 [00:00<00:00, 65.28it/s]\n",
      "Epoch 443/1000 - Loss: 5.938398: 100%|██████████| 1/1 [00:00<00:00, 61.96it/s]\n",
      "Epoch 444/1000 - Loss: 5.937284: 100%|██████████| 1/1 [00:00<00:00, 52.31it/s]\n",
      "Epoch 445/1000 - Loss: 5.938443: 100%|██████████| 1/1 [00:00<00:00, 62.35it/s]\n",
      "Epoch 446/1000 - Loss: 5.939088: 100%|██████████| 1/1 [00:00<00:00, 64.32it/s]\n",
      "Epoch 447/1000 - Loss: 5.937110: 100%|██████████| 1/1 [00:00<00:00, 62.38it/s]\n",
      "Epoch 448/1000 - Loss: 5.937637: 100%|██████████| 1/1 [00:00<00:00, 61.64it/s]\n",
      "Epoch 449/1000 - Loss: 5.938124: 100%|██████████| 1/1 [00:00<00:00, 61.54it/s]\n",
      "Epoch 450/1000 - Loss: 5.938290: 100%|██████████| 1/1 [00:00<00:00, 61.15it/s]\n",
      "Epoch 451/1000 - Loss: 5.937922: 100%|██████████| 1/1 [00:00<00:00, 63.10it/s]\n",
      "Epoch 452/1000 - Loss: 5.939324: 100%|██████████| 1/1 [00:00<00:00, 64.18it/s]\n",
      "Epoch 453/1000 - Loss: 5.937966: 100%|██████████| 1/1 [00:00<00:00, 61.75it/s]\n",
      "Epoch 454/1000 - Loss: 5.937126: 100%|██████████| 1/1 [00:00<00:00, 62.28it/s]\n",
      "Epoch 455/1000 - Loss: 5.938054: 100%|██████████| 1/1 [00:00<00:00, 63.05it/s]\n",
      "Epoch 456/1000 - Loss: 5.938175: 100%|██████████| 1/1 [00:00<00:00, 62.44it/s]\n",
      "Epoch 457/1000 - Loss: 5.937723: 100%|██████████| 1/1 [00:00<00:00, 64.70it/s]\n",
      "Epoch 458/1000 - Loss: 5.937068: 100%|██████████| 1/1 [00:00<00:00, 63.17it/s]\n",
      "Epoch 459/1000 - Loss: 5.936450: 100%|██████████| 1/1 [00:00<00:00, 54.59it/s]\n",
      "Epoch 460/1000 - Loss: 5.937735: 100%|██████████| 1/1 [00:00<00:00, 62.30it/s]\n",
      "Epoch 461/1000 - Loss: 5.937488: 100%|██████████| 1/1 [00:00<00:00, 61.71it/s]\n",
      "Epoch 462/1000 - Loss: 5.937561: 100%|██████████| 1/1 [00:00<00:00, 57.48it/s]\n",
      "Epoch 463/1000 - Loss: 5.936693: 100%|██████████| 1/1 [00:00<00:00, 61.47it/s]\n",
      "Epoch 464/1000 - Loss: 5.936351: 100%|██████████| 1/1 [00:00<00:00, 60.25it/s]\n",
      "Epoch 465/1000 - Loss: 5.937414: 100%|██████████| 1/1 [00:00<00:00, 58.90it/s]\n",
      "Epoch 466/1000 - Loss: 5.937744: 100%|██████████| 1/1 [00:00<00:00, 56.97it/s]\n",
      "Epoch 467/1000 - Loss: 5.937010: 100%|██████████| 1/1 [00:00<00:00, 62.71it/s]\n",
      "Epoch 468/1000 - Loss: 5.937431: 100%|██████████| 1/1 [00:00<00:00, 65.69it/s]\n",
      "Epoch 469/1000 - Loss: 5.935102: 100%|██████████| 1/1 [00:00<00:00, 62.26it/s]\n",
      "Epoch 470/1000 - Loss: 5.936543: 100%|██████████| 1/1 [00:00<00:00, 58.03it/s]\n",
      "Epoch 471/1000 - Loss: 5.937160: 100%|██████████| 1/1 [00:00<00:00, 62.81it/s]\n",
      "Epoch 472/1000 - Loss: 5.937014: 100%|██████████| 1/1 [00:00<00:00, 63.13it/s]\n",
      "Epoch 473/1000 - Loss: 5.936558: 100%|██████████| 1/1 [00:00<00:00, 62.80it/s]\n",
      "Epoch 474/1000 - Loss: 5.936563: 100%|██████████| 1/1 [00:00<00:00, 59.85it/s]\n",
      "Epoch 475/1000 - Loss: 5.936276: 100%|██████████| 1/1 [00:00<00:00, 62.34it/s]\n",
      "Epoch 476/1000 - Loss: 5.936829: 100%|██████████| 1/1 [00:00<00:00, 62.71it/s]\n",
      "Epoch 477/1000 - Loss: 5.936609: 100%|██████████| 1/1 [00:00<00:00, 61.36it/s]\n",
      "Epoch 478/1000 - Loss: 5.935703: 100%|██████████| 1/1 [00:00<00:00, 59.19it/s]\n",
      "Epoch 479/1000 - Loss: 5.936396: 100%|██████████| 1/1 [00:00<00:00, 60.35it/s]\n",
      "Epoch 480/1000 - Loss: 5.935916: 100%|██████████| 1/1 [00:00<00:00, 62.39it/s]\n",
      "Epoch 481/1000 - Loss: 5.936123: 100%|██████████| 1/1 [00:00<00:00, 58.53it/s]\n",
      "Epoch 482/1000 - Loss: 5.935146: 100%|██████████| 1/1 [00:00<00:00, 58.45it/s]\n",
      "Epoch 483/1000 - Loss: 5.935958: 100%|██████████| 1/1 [00:00<00:00, 58.98it/s]\n",
      "Epoch 484/1000 - Loss: 5.936203: 100%|██████████| 1/1 [00:00<00:00, 63.69it/s]\n",
      "Epoch 485/1000 - Loss: 5.935703: 100%|██████████| 1/1 [00:00<00:00, 62.40it/s]\n",
      "Epoch 486/1000 - Loss: 5.936192: 100%|██████████| 1/1 [00:00<00:00, 59.20it/s]\n",
      "Epoch 487/1000 - Loss: 5.935632: 100%|██████████| 1/1 [00:00<00:00, 66.02it/s]\n",
      "Epoch 488/1000 - Loss: 5.935565: 100%|██████████| 1/1 [00:00<00:00, 60.83it/s]\n",
      "Epoch 489/1000 - Loss: 5.935616: 100%|██████████| 1/1 [00:00<00:00, 63.33it/s]\n",
      "Epoch 490/1000 - Loss: 5.934993: 100%|██████████| 1/1 [00:00<00:00, 58.86it/s]\n",
      "Epoch 491/1000 - Loss: 5.933267: 100%|██████████| 1/1 [00:00<00:00, 59.05it/s]\n",
      "Epoch 492/1000 - Loss: 5.934934: 100%|██████████| 1/1 [00:00<00:00, 59.51it/s]\n",
      "Epoch 493/1000 - Loss: 5.935261: 100%|██████████| 1/1 [00:00<00:00, 62.19it/s]\n",
      "Epoch 494/1000 - Loss: 5.935188: 100%|██████████| 1/1 [00:00<00:00, 64.48it/s]\n",
      "Epoch 495/1000 - Loss: 5.935068: 100%|██████████| 1/1 [00:00<00:00, 60.08it/s]\n",
      "Epoch 496/1000 - Loss: 5.935592: 100%|██████████| 1/1 [00:00<00:00, 64.62it/s]\n",
      "Epoch 497/1000 - Loss: 5.932720: 100%|██████████| 1/1 [00:00<00:00, 62.40it/s]\n",
      "Epoch 498/1000 - Loss: 5.935193: 100%|██████████| 1/1 [00:00<00:00, 64.07it/s]\n",
      "Epoch 499/1000 - Loss: 5.933982: 100%|██████████| 1/1 [00:00<00:00, 64.08it/s]\n",
      "Epoch 500/1000 - Loss: 5.935009: 100%|██████████| 1/1 [00:00<00:00, 63.13it/s]\n",
      "Epoch 501/1000 - Loss: 5.932788: 100%|██████████| 1/1 [00:00<00:00, 62.96it/s]\n",
      "Epoch 502/1000 - Loss: 5.932982: 100%|██████████| 1/1 [00:00<00:00, 64.58it/s]\n",
      "Epoch 503/1000 - Loss: 5.932994: 100%|██████████| 1/1 [00:00<00:00, 64.44it/s]\n",
      "Epoch 504/1000 - Loss: 5.933540: 100%|██████████| 1/1 [00:00<00:00, 67.47it/s]\n",
      "Epoch 505/1000 - Loss: 5.934665: 100%|██████████| 1/1 [00:00<00:00, 62.67it/s]\n",
      "Epoch 506/1000 - Loss: 5.934755: 100%|██████████| 1/1 [00:00<00:00, 65.71it/s]\n",
      "Epoch 507/1000 - Loss: 5.934695: 100%|██████████| 1/1 [00:00<00:00, 59.96it/s]\n",
      "Epoch 508/1000 - Loss: 5.934257: 100%|██████████| 1/1 [00:00<00:00, 56.77it/s]\n",
      "Epoch 509/1000 - Loss: 5.934247: 100%|██████████| 1/1 [00:00<00:00, 64.17it/s]\n",
      "Epoch 510/1000 - Loss: 5.934210: 100%|██████████| 1/1 [00:00<00:00, 59.61it/s]\n",
      "Epoch 511/1000 - Loss: 5.934953: 100%|██████████| 1/1 [00:00<00:00, 65.17it/s]\n",
      "Epoch 512/1000 - Loss: 5.932287: 100%|██████████| 1/1 [00:00<00:00, 66.00it/s]\n",
      "Epoch 513/1000 - Loss: 5.934155: 100%|██████████| 1/1 [00:00<00:00, 64.90it/s]\n",
      "Epoch 514/1000 - Loss: 5.933731: 100%|██████████| 1/1 [00:00<00:00, 62.08it/s]\n",
      "Epoch 515/1000 - Loss: 5.932785: 100%|██████████| 1/1 [00:00<00:00, 65.86it/s]\n",
      "Epoch 516/1000 - Loss: 5.931098: 100%|██████████| 1/1 [00:00<00:00, 63.41it/s]\n",
      "Epoch 517/1000 - Loss: 5.933025: 100%|██████████| 1/1 [00:00<00:00, 63.19it/s]\n",
      "Epoch 518/1000 - Loss: 5.932303: 100%|██████████| 1/1 [00:00<00:00, 62.45it/s]\n",
      "Epoch 519/1000 - Loss: 5.933610: 100%|██████████| 1/1 [00:00<00:00, 62.73it/s]\n",
      "Epoch 520/1000 - Loss: 5.931781: 100%|██████████| 1/1 [00:00<00:00, 64.32it/s]\n",
      "Epoch 521/1000 - Loss: 5.932895: 100%|██████████| 1/1 [00:00<00:00, 60.37it/s]\n",
      "Epoch 522/1000 - Loss: 5.935392: 100%|██████████| 1/1 [00:00<00:00, 55.27it/s]\n",
      "Epoch 523/1000 - Loss: 5.933817: 100%|██████████| 1/1 [00:00<00:00, 59.55it/s]\n",
      "Epoch 524/1000 - Loss: 5.933597: 100%|██████████| 1/1 [00:00<00:00, 56.24it/s]\n",
      "Epoch 525/1000 - Loss: 5.933723: 100%|██████████| 1/1 [00:00<00:00, 67.33it/s]\n",
      "Epoch 526/1000 - Loss: 5.931989: 100%|██████████| 1/1 [00:00<00:00, 61.66it/s]\n",
      "Epoch 527/1000 - Loss: 5.933165: 100%|██████████| 1/1 [00:00<00:00, 61.06it/s]\n",
      "Epoch 528/1000 - Loss: 5.930479: 100%|██████████| 1/1 [00:00<00:00, 63.90it/s]\n",
      "Epoch 529/1000 - Loss: 5.933596: 100%|██████████| 1/1 [00:00<00:00, 66.82it/s]\n",
      "Epoch 530/1000 - Loss: 5.933767: 100%|██████████| 1/1 [00:00<00:00, 65.51it/s]\n",
      "Epoch 531/1000 - Loss: 5.932752: 100%|██████████| 1/1 [00:00<00:00, 59.36it/s]\n",
      "Epoch 532/1000 - Loss: 5.927652: 100%|██████████| 1/1 [00:00<00:00, 62.24it/s]\n",
      "Epoch 533/1000 - Loss: 5.931746: 100%|██████████| 1/1 [00:00<00:00, 65.37it/s]\n",
      "Epoch 534/1000 - Loss: 5.931157: 100%|██████████| 1/1 [00:00<00:00, 63.84it/s]\n",
      "Epoch 535/1000 - Loss: 5.932627: 100%|██████████| 1/1 [00:00<00:00, 58.04it/s]\n",
      "Epoch 536/1000 - Loss: 5.932106: 100%|██████████| 1/1 [00:00<00:00, 63.20it/s]\n",
      "Epoch 537/1000 - Loss: 5.931514: 100%|██████████| 1/1 [00:00<00:00, 63.16it/s]\n",
      "Epoch 538/1000 - Loss: 5.932126: 100%|██████████| 1/1 [00:00<00:00, 57.64it/s]\n",
      "Epoch 539/1000 - Loss: 5.931188: 100%|██████████| 1/1 [00:00<00:00, 64.54it/s]\n",
      "Epoch 540/1000 - Loss: 5.934419: 100%|██████████| 1/1 [00:00<00:00, 59.92it/s]\n",
      "Epoch 541/1000 - Loss: 5.931166: 100%|██████████| 1/1 [00:00<00:00, 60.74it/s]\n",
      "Epoch 542/1000 - Loss: 5.925376: 100%|██████████| 1/1 [00:00<00:00, 63.62it/s]\n",
      "Epoch 543/1000 - Loss: 5.933735: 100%|██████████| 1/1 [00:00<00:00, 73.39it/s]\n",
      "Epoch 544/1000 - Loss: 5.933120: 100%|██████████| 1/1 [00:00<00:00, 62.87it/s]\n",
      "Epoch 545/1000 - Loss: 5.932832: 100%|██████████| 1/1 [00:00<00:00, 58.93it/s]\n",
      "Epoch 546/1000 - Loss: 5.932366: 100%|██████████| 1/1 [00:00<00:00, 59.40it/s]\n",
      "Epoch 547/1000 - Loss: 5.932574: 100%|██████████| 1/1 [00:00<00:00, 64.36it/s]\n",
      "Epoch 548/1000 - Loss: 5.932814: 100%|██████████| 1/1 [00:00<00:00, 59.33it/s]\n",
      "Epoch 549/1000 - Loss: 5.932857: 100%|██████████| 1/1 [00:00<00:00, 61.34it/s]\n",
      "Epoch 550/1000 - Loss: 5.930402: 100%|██████████| 1/1 [00:00<00:00, 60.80it/s]\n",
      "Epoch 551/1000 - Loss: 5.931159: 100%|██████████| 1/1 [00:00<00:00, 60.79it/s]\n",
      "Epoch 552/1000 - Loss: 5.932772: 100%|██████████| 1/1 [00:00<00:00, 59.57it/s]\n",
      "Epoch 553/1000 - Loss: 5.928870: 100%|██████████| 1/1 [00:00<00:00, 58.68it/s]\n",
      "Epoch 554/1000 - Loss: 5.930045: 100%|██████████| 1/1 [00:00<00:00, 58.55it/s]\n",
      "Epoch 555/1000 - Loss: 5.931189: 100%|██████████| 1/1 [00:00<00:00, 56.66it/s]\n",
      "Epoch 556/1000 - Loss: 5.930748: 100%|██████████| 1/1 [00:00<00:00, 58.86it/s]\n",
      "Epoch 557/1000 - Loss: 5.930688: 100%|██████████| 1/1 [00:00<00:00, 60.46it/s]\n",
      "Epoch 558/1000 - Loss: 5.930873: 100%|██████████| 1/1 [00:00<00:00, 60.33it/s]\n",
      "Epoch 559/1000 - Loss: 5.930932: 100%|██████████| 1/1 [00:00<00:00, 59.80it/s]\n",
      "Epoch 560/1000 - Loss: 5.931267: 100%|██████████| 1/1 [00:00<00:00, 62.62it/s]\n",
      "Epoch 561/1000 - Loss: 5.929338: 100%|██████████| 1/1 [00:00<00:00, 59.82it/s]\n",
      "Epoch 562/1000 - Loss: 5.930511: 100%|██████████| 1/1 [00:00<00:00, 55.21it/s]\n",
      "Epoch 563/1000 - Loss: 5.929943: 100%|██████████| 1/1 [00:00<00:00, 62.24it/s]\n",
      "Epoch 564/1000 - Loss: 5.930005: 100%|██████████| 1/1 [00:00<00:00, 64.36it/s]\n",
      "Epoch 565/1000 - Loss: 5.927127: 100%|██████████| 1/1 [00:00<00:00, 61.37it/s]\n",
      "Epoch 566/1000 - Loss: 5.928178: 100%|██████████| 1/1 [00:00<00:00, 61.55it/s]\n",
      "Epoch 567/1000 - Loss: 5.929299: 100%|██████████| 1/1 [00:00<00:00, 63.47it/s]\n",
      "Epoch 568/1000 - Loss: 5.925906: 100%|██████████| 1/1 [00:00<00:00, 62.44it/s]\n",
      "Epoch 569/1000 - Loss: 5.929507: 100%|██████████| 1/1 [00:00<00:00, 60.88it/s]\n",
      "Epoch 570/1000 - Loss: 5.929584: 100%|██████████| 1/1 [00:00<00:00, 60.31it/s]\n",
      "Epoch 571/1000 - Loss: 5.929120: 100%|██████████| 1/1 [00:00<00:00, 60.87it/s]\n",
      "Epoch 572/1000 - Loss: 5.929513: 100%|██████████| 1/1 [00:00<00:00, 57.38it/s]\n",
      "Epoch 573/1000 - Loss: 5.929628: 100%|██████████| 1/1 [00:00<00:00, 57.74it/s]\n",
      "Epoch 574/1000 - Loss: 5.928782: 100%|██████████| 1/1 [00:00<00:00, 59.92it/s]\n",
      "Epoch 575/1000 - Loss: 5.926460: 100%|██████████| 1/1 [00:00<00:00, 60.95it/s]\n",
      "Epoch 576/1000 - Loss: 5.929135: 100%|██████████| 1/1 [00:00<00:00, 60.99it/s]\n",
      "Epoch 577/1000 - Loss: 5.928013: 100%|██████████| 1/1 [00:00<00:00, 58.67it/s]\n",
      "Epoch 578/1000 - Loss: 5.927516: 100%|██████████| 1/1 [00:00<00:00, 54.48it/s]\n",
      "Epoch 579/1000 - Loss: 5.928914: 100%|██████████| 1/1 [00:00<00:00, 58.59it/s]\n",
      "Epoch 580/1000 - Loss: 5.928674: 100%|██████████| 1/1 [00:00<00:00, 63.66it/s]\n",
      "Epoch 581/1000 - Loss: 5.928067: 100%|██████████| 1/1 [00:00<00:00, 60.47it/s]\n",
      "Epoch 582/1000 - Loss: 5.928894: 100%|██████████| 1/1 [00:00<00:00, 61.51it/s]\n",
      "Epoch 583/1000 - Loss: 5.929133: 100%|██████████| 1/1 [00:00<00:00, 59.25it/s]\n",
      "Epoch 584/1000 - Loss: 5.923746: 100%|██████████| 1/1 [00:00<00:00, 60.78it/s]\n",
      "Epoch 585/1000 - Loss: 5.927267: 100%|██████████| 1/1 [00:00<00:00, 63.39it/s]\n",
      "Epoch 586/1000 - Loss: 5.927222: 100%|██████████| 1/1 [00:00<00:00, 54.18it/s]\n",
      "Epoch 587/1000 - Loss: 5.927557: 100%|██████████| 1/1 [00:00<00:00, 62.66it/s]\n",
      "Epoch 588/1000 - Loss: 5.929928: 100%|██████████| 1/1 [00:00<00:00, 58.58it/s]\n",
      "Epoch 589/1000 - Loss: 5.926767: 100%|██████████| 1/1 [00:00<00:00, 62.15it/s]\n",
      "Epoch 590/1000 - Loss: 5.929036: 100%|██████████| 1/1 [00:00<00:00, 59.73it/s]\n",
      "Epoch 591/1000 - Loss: 5.927851: 100%|██████████| 1/1 [00:00<00:00, 63.57it/s]\n",
      "Epoch 592/1000 - Loss: 5.926861: 100%|██████████| 1/1 [00:00<00:00, 65.93it/s]\n",
      "Epoch 593/1000 - Loss: 5.927103: 100%|██████████| 1/1 [00:00<00:00, 62.78it/s]\n",
      "Epoch 594/1000 - Loss: 5.927238: 100%|██████████| 1/1 [00:00<00:00, 62.02it/s]\n",
      "Epoch 595/1000 - Loss: 5.923554: 100%|██████████| 1/1 [00:00<00:00, 60.99it/s]\n",
      "Epoch 596/1000 - Loss: 5.927435: 100%|██████████| 1/1 [00:00<00:00, 60.63it/s]\n",
      "Epoch 597/1000 - Loss: 5.924362: 100%|██████████| 1/1 [00:00<00:00, 64.97it/s]\n",
      "Epoch 598/1000 - Loss: 5.927016: 100%|██████████| 1/1 [00:00<00:00, 67.83it/s]\n",
      "Epoch 599/1000 - Loss: 5.924705: 100%|██████████| 1/1 [00:00<00:00, 66.17it/s]\n",
      "Epoch 600/1000 - Loss: 5.926935: 100%|██████████| 1/1 [00:00<00:00, 68.58it/s]\n",
      "Epoch 601/1000 - Loss: 5.927959: 100%|██████████| 1/1 [00:00<00:00, 71.17it/s]\n",
      "Epoch 602/1000 - Loss: 5.925497: 100%|██████████| 1/1 [00:00<00:00, 70.42it/s]\n",
      "Epoch 603/1000 - Loss: 5.925777: 100%|██████████| 1/1 [00:00<00:00, 62.51it/s]\n",
      "Epoch 604/1000 - Loss: 5.927730: 100%|██████████| 1/1 [00:00<00:00, 62.89it/s]\n",
      "Epoch 605/1000 - Loss: 5.926743: 100%|██████████| 1/1 [00:00<00:00, 65.41it/s]\n",
      "Epoch 606/1000 - Loss: 5.927283: 100%|██████████| 1/1 [00:00<00:00, 63.77it/s]\n",
      "Epoch 607/1000 - Loss: 5.927539: 100%|██████████| 1/1 [00:00<00:00, 73.08it/s]\n",
      "Epoch 608/1000 - Loss: 5.925677: 100%|██████████| 1/1 [00:00<00:00, 69.58it/s]\n",
      "Epoch 609/1000 - Loss: 5.925702: 100%|██████████| 1/1 [00:00<00:00, 64.65it/s]\n",
      "Epoch 610/1000 - Loss: 5.924623: 100%|██████████| 1/1 [00:00<00:00, 66.58it/s]\n",
      "Epoch 611/1000 - Loss: 5.924920: 100%|██████████| 1/1 [00:00<00:00, 63.81it/s]\n",
      "Epoch 612/1000 - Loss: 5.925209: 100%|██████████| 1/1 [00:00<00:00, 63.67it/s]\n",
      "Epoch 613/1000 - Loss: 5.924772: 100%|██████████| 1/1 [00:00<00:00, 58.56it/s]\n",
      "Epoch 614/1000 - Loss: 5.925232: 100%|██████████| 1/1 [00:00<00:00, 61.23it/s]\n",
      "Epoch 615/1000 - Loss: 5.924719: 100%|██████████| 1/1 [00:00<00:00, 65.91it/s]\n",
      "Epoch 616/1000 - Loss: 5.922120: 100%|██████████| 1/1 [00:00<00:00, 68.12it/s]\n",
      "Epoch 617/1000 - Loss: 5.924676: 100%|██████████| 1/1 [00:00<00:00, 64.39it/s]\n",
      "Epoch 618/1000 - Loss: 5.924606: 100%|██████████| 1/1 [00:00<00:00, 59.55it/s]\n",
      "Epoch 619/1000 - Loss: 5.923514: 100%|██████████| 1/1 [00:00<00:00, 59.31it/s]\n",
      "Epoch 620/1000 - Loss: 5.924205: 100%|██████████| 1/1 [00:00<00:00, 59.67it/s]\n",
      "Epoch 621/1000 - Loss: 5.925504: 100%|██████████| 1/1 [00:00<00:00, 60.08it/s]\n",
      "Epoch 622/1000 - Loss: 5.925023: 100%|██████████| 1/1 [00:00<00:00, 57.77it/s]\n",
      "Epoch 623/1000 - Loss: 5.924692: 100%|██████████| 1/1 [00:00<00:00, 59.40it/s]\n",
      "Epoch 624/1000 - Loss: 5.924235: 100%|██████████| 1/1 [00:00<00:00, 55.79it/s]\n",
      "Epoch 625/1000 - Loss: 5.923777: 100%|██████████| 1/1 [00:00<00:00, 61.60it/s]\n",
      "Epoch 626/1000 - Loss: 5.922842: 100%|██████████| 1/1 [00:00<00:00, 61.11it/s]\n",
      "Epoch 627/1000 - Loss: 5.924172: 100%|██████████| 1/1 [00:00<00:00, 55.97it/s]\n",
      "Epoch 628/1000 - Loss: 5.923717: 100%|██████████| 1/1 [00:00<00:00, 58.97it/s]\n",
      "Epoch 629/1000 - Loss: 5.920233: 100%|██████████| 1/1 [00:00<00:00, 63.51it/s]\n",
      "Epoch 630/1000 - Loss: 5.922602: 100%|██████████| 1/1 [00:00<00:00, 59.45it/s]\n",
      "Epoch 631/1000 - Loss: 5.924234: 100%|██████████| 1/1 [00:00<00:00, 44.42it/s]\n",
      "Epoch 632/1000 - Loss: 5.924247: 100%|██████████| 1/1 [00:00<00:00, 55.63it/s]\n",
      "Epoch 633/1000 - Loss: 5.924365: 100%|██████████| 1/1 [00:00<00:00, 67.28it/s]\n",
      "Epoch 634/1000 - Loss: 5.922643: 100%|██████████| 1/1 [00:00<00:00, 75.16it/s]\n",
      "Epoch 635/1000 - Loss: 5.921429: 100%|██████████| 1/1 [00:00<00:00, 68.24it/s]\n",
      "Epoch 636/1000 - Loss: 5.921702: 100%|██████████| 1/1 [00:00<00:00, 72.00it/s]\n",
      "Epoch 637/1000 - Loss: 5.921939: 100%|██████████| 1/1 [00:00<00:00, 73.07it/s]\n",
      "Epoch 638/1000 - Loss: 5.925282: 100%|██████████| 1/1 [00:00<00:00, 64.68it/s]\n",
      "Epoch 639/1000 - Loss: 5.923071: 100%|██████████| 1/1 [00:00<00:00, 68.57it/s]\n",
      "Epoch 640/1000 - Loss: 5.920932: 100%|██████████| 1/1 [00:00<00:00, 59.78it/s]\n",
      "Epoch 641/1000 - Loss: 5.921460: 100%|██████████| 1/1 [00:00<00:00, 57.19it/s]\n",
      "Epoch 642/1000 - Loss: 5.923903: 100%|██████████| 1/1 [00:00<00:00, 62.41it/s]\n",
      "Epoch 643/1000 - Loss: 5.921401: 100%|██████████| 1/1 [00:00<00:00, 58.68it/s]\n",
      "Epoch 644/1000 - Loss: 5.922443: 100%|██████████| 1/1 [00:00<00:00, 59.73it/s]\n",
      "Epoch 645/1000 - Loss: 5.922048: 100%|██████████| 1/1 [00:00<00:00, 60.48it/s]\n",
      "Epoch 646/1000 - Loss: 5.922210: 100%|██████████| 1/1 [00:00<00:00, 56.75it/s]\n",
      "Epoch 647/1000 - Loss: 5.922138: 100%|██████████| 1/1 [00:00<00:00, 59.34it/s]\n",
      "Epoch 648/1000 - Loss: 5.919718: 100%|██████████| 1/1 [00:00<00:00, 55.45it/s]\n",
      "Epoch 649/1000 - Loss: 5.924533: 100%|██████████| 1/1 [00:00<00:00, 59.08it/s]\n",
      "Epoch 650/1000 - Loss: 5.921590: 100%|██████████| 1/1 [00:00<00:00, 56.54it/s]\n",
      "Epoch 651/1000 - Loss: 5.921964: 100%|██████████| 1/1 [00:00<00:00, 61.86it/s]\n",
      "Epoch 652/1000 - Loss: 5.919232: 100%|██████████| 1/1 [00:00<00:00, 58.07it/s]\n",
      "Epoch 653/1000 - Loss: 5.922972: 100%|██████████| 1/1 [00:00<00:00, 62.10it/s]\n",
      "Epoch 654/1000 - Loss: 5.919575: 100%|██████████| 1/1 [00:00<00:00, 73.75it/s]\n",
      "Epoch 655/1000 - Loss: 5.917334: 100%|██████████| 1/1 [00:00<00:00, 78.42it/s]\n",
      "Epoch 656/1000 - Loss: 5.919465: 100%|██████████| 1/1 [00:00<00:00, 75.42it/s]\n",
      "Epoch 657/1000 - Loss: 5.921531: 100%|██████████| 1/1 [00:00<00:00, 83.92it/s]\n",
      "Epoch 658/1000 - Loss: 5.920193: 100%|██████████| 1/1 [00:00<00:00, 82.50it/s]\n",
      "Epoch 659/1000 - Loss: 5.919389: 100%|██████████| 1/1 [00:00<00:00, 68.59it/s]\n",
      "Epoch 660/1000 - Loss: 5.919636: 100%|██████████| 1/1 [00:00<00:00, 72.49it/s]\n",
      "Epoch 661/1000 - Loss: 5.922636: 100%|██████████| 1/1 [00:00<00:00, 69.59it/s]\n",
      "Epoch 662/1000 - Loss: 5.921583: 100%|██████████| 1/1 [00:00<00:00, 63.29it/s]\n",
      "Epoch 663/1000 - Loss: 5.921058: 100%|██████████| 1/1 [00:00<00:00, 66.21it/s]\n",
      "Epoch 664/1000 - Loss: 5.917936: 100%|██████████| 1/1 [00:00<00:00, 65.65it/s]\n",
      "Epoch 665/1000 - Loss: 5.919539: 100%|██████████| 1/1 [00:00<00:00, 58.22it/s]\n",
      "Epoch 666/1000 - Loss: 5.920121: 100%|██████████| 1/1 [00:00<00:00, 59.59it/s]\n",
      "Epoch 667/1000 - Loss: 5.920511: 100%|██████████| 1/1 [00:00<00:00, 57.06it/s]\n",
      "Epoch 668/1000 - Loss: 5.918540: 100%|██████████| 1/1 [00:00<00:00, 59.99it/s]\n",
      "Epoch 669/1000 - Loss: 5.920699: 100%|██████████| 1/1 [00:00<00:00, 59.71it/s]\n",
      "Epoch 670/1000 - Loss: 5.918440: 100%|██████████| 1/1 [00:00<00:00, 63.78it/s]\n",
      "Epoch 671/1000 - Loss: 5.919189: 100%|██████████| 1/1 [00:00<00:00, 63.73it/s]\n",
      "Epoch 672/1000 - Loss: 5.919702: 100%|██████████| 1/1 [00:00<00:00, 59.41it/s]\n",
      "Epoch 673/1000 - Loss: 5.919507: 100%|██████████| 1/1 [00:00<00:00, 62.71it/s]\n",
      "Epoch 674/1000 - Loss: 5.916998: 100%|██████████| 1/1 [00:00<00:00, 58.11it/s]\n",
      "Epoch 675/1000 - Loss: 5.918227: 100%|██████████| 1/1 [00:00<00:00, 59.83it/s]\n",
      "Epoch 676/1000 - Loss: 5.918682: 100%|██████████| 1/1 [00:00<00:00, 63.33it/s]\n",
      "Epoch 677/1000 - Loss: 5.916340: 100%|██████████| 1/1 [00:00<00:00, 64.05it/s]\n",
      "Epoch 678/1000 - Loss: 5.917024: 100%|██████████| 1/1 [00:00<00:00, 63.43it/s]\n",
      "Epoch 679/1000 - Loss: 5.924192: 100%|██████████| 1/1 [00:00<00:00, 62.04it/s]\n",
      "Epoch 680/1000 - Loss: 5.916743: 100%|██████████| 1/1 [00:00<00:00, 61.49it/s]\n",
      "Epoch 681/1000 - Loss: 5.917032: 100%|██████████| 1/1 [00:00<00:00, 62.81it/s]\n",
      "Epoch 682/1000 - Loss: 5.917799: 100%|██████████| 1/1 [00:00<00:00, 64.78it/s]\n",
      "Epoch 683/1000 - Loss: 5.930176: 100%|██████████| 1/1 [00:00<00:00, 60.12it/s]\n",
      "Epoch 684/1000 - Loss: 5.916362: 100%|██████████| 1/1 [00:00<00:00, 62.54it/s]\n",
      "Epoch 685/1000 - Loss: 5.919206: 100%|██████████| 1/1 [00:00<00:00, 62.02it/s]\n",
      "Epoch 686/1000 - Loss: 5.920607: 100%|██████████| 1/1 [00:00<00:00, 55.83it/s]\n",
      "Epoch 687/1000 - Loss: 5.915499: 100%|██████████| 1/1 [00:00<00:00, 58.91it/s]\n",
      "Epoch 688/1000 - Loss: 5.914630: 100%|██████████| 1/1 [00:00<00:00, 64.51it/s]\n",
      "Epoch 689/1000 - Loss: 5.917574: 100%|██████████| 1/1 [00:00<00:00, 57.69it/s]\n",
      "Epoch 690/1000 - Loss: 5.917656: 100%|██████████| 1/1 [00:00<00:00, 65.04it/s]\n",
      "Epoch 691/1000 - Loss: 5.917637: 100%|██████████| 1/1 [00:00<00:00, 64.34it/s]\n",
      "Epoch 692/1000 - Loss: 5.916445: 100%|██████████| 1/1 [00:00<00:00, 60.93it/s]\n",
      "Epoch 693/1000 - Loss: 5.919900: 100%|██████████| 1/1 [00:00<00:00, 59.69it/s]\n",
      "Epoch 694/1000 - Loss: 5.919172: 100%|██████████| 1/1 [00:00<00:00, 62.34it/s]\n",
      "Epoch 695/1000 - Loss: 5.916552: 100%|██████████| 1/1 [00:00<00:00, 53.36it/s]\n",
      "Epoch 696/1000 - Loss: 5.919374: 100%|██████████| 1/1 [00:00<00:00, 60.04it/s]\n",
      "Epoch 697/1000 - Loss: 5.917838: 100%|██████████| 1/1 [00:00<00:00, 58.04it/s]\n",
      "Epoch 698/1000 - Loss: 5.917660: 100%|██████████| 1/1 [00:00<00:00, 60.96it/s]\n",
      "Epoch 699/1000 - Loss: 5.916498: 100%|██████████| 1/1 [00:00<00:00, 59.02it/s]\n",
      "Epoch 700/1000 - Loss: 5.917006: 100%|██████████| 1/1 [00:00<00:00, 61.40it/s]\n",
      "Epoch 701/1000 - Loss: 5.916208: 100%|██████████| 1/1 [00:00<00:00, 60.52it/s]\n",
      "Epoch 702/1000 - Loss: 5.916707: 100%|██████████| 1/1 [00:00<00:00, 61.49it/s]\n",
      "Epoch 703/1000 - Loss: 5.914399: 100%|██████████| 1/1 [00:00<00:00, 64.67it/s]\n",
      "Epoch 704/1000 - Loss: 5.915385: 100%|██████████| 1/1 [00:00<00:00, 51.71it/s]\n",
      "Epoch 705/1000 - Loss: 5.915468: 100%|██████████| 1/1 [00:00<00:00, 66.84it/s]\n",
      "Epoch 706/1000 - Loss: 5.916746: 100%|██████████| 1/1 [00:00<00:00, 57.65it/s]\n",
      "Epoch 707/1000 - Loss: 5.914244: 100%|██████████| 1/1 [00:00<00:00, 63.52it/s]\n",
      "Epoch 708/1000 - Loss: 5.914840: 100%|██████████| 1/1 [00:00<00:00, 62.44it/s]\n",
      "Epoch 709/1000 - Loss: 5.912570: 100%|██████████| 1/1 [00:00<00:00, 63.04it/s]\n",
      "Epoch 710/1000 - Loss: 5.912391: 100%|██████████| 1/1 [00:00<00:00, 58.70it/s]\n",
      "Epoch 711/1000 - Loss: 5.915202: 100%|██████████| 1/1 [00:00<00:00, 64.12it/s]\n",
      "Epoch 712/1000 - Loss: 5.915356: 100%|██████████| 1/1 [00:00<00:00, 65.75it/s]\n",
      "Epoch 713/1000 - Loss: 5.911663: 100%|██████████| 1/1 [00:00<00:00, 63.31it/s]\n",
      "Epoch 714/1000 - Loss: 5.913799: 100%|██████████| 1/1 [00:00<00:00, 62.90it/s]\n",
      "Epoch 715/1000 - Loss: 5.915457: 100%|██████████| 1/1 [00:00<00:00, 63.75it/s]\n",
      "Epoch 716/1000 - Loss: 5.914640: 100%|██████████| 1/1 [00:00<00:00, 52.25it/s]\n",
      "Epoch 717/1000 - Loss: 5.912308: 100%|██████████| 1/1 [00:00<00:00, 64.11it/s]\n",
      "Epoch 718/1000 - Loss: 5.911969: 100%|██████████| 1/1 [00:00<00:00, 57.57it/s]\n",
      "Epoch 719/1000 - Loss: 5.910601: 100%|██████████| 1/1 [00:00<00:00, 58.18it/s]\n",
      "Epoch 720/1000 - Loss: 5.912199: 100%|██████████| 1/1 [00:00<00:00, 63.33it/s]\n",
      "Epoch 721/1000 - Loss: 5.914390: 100%|██████████| 1/1 [00:00<00:00, 64.71it/s]\n",
      "Epoch 722/1000 - Loss: 5.910873: 100%|██████████| 1/1 [00:00<00:00, 68.80it/s]\n",
      "Epoch 723/1000 - Loss: 5.912166: 100%|██████████| 1/1 [00:00<00:00, 65.09it/s]\n",
      "Epoch 724/1000 - Loss: 5.910767: 100%|██████████| 1/1 [00:00<00:00, 65.59it/s]\n",
      "Epoch 725/1000 - Loss: 5.912547: 100%|██████████| 1/1 [00:00<00:00, 55.16it/s]\n",
      "Epoch 726/1000 - Loss: 5.909580: 100%|██████████| 1/1 [00:00<00:00, 60.11it/s]\n",
      "Epoch 727/1000 - Loss: 5.911662: 100%|██████████| 1/1 [00:00<00:00, 58.56it/s]\n",
      "Epoch 728/1000 - Loss: 5.910961: 100%|██████████| 1/1 [00:00<00:00, 64.57it/s]\n",
      "Epoch 729/1000 - Loss: 5.913135: 100%|██████████| 1/1 [00:00<00:00, 67.70it/s]\n",
      "Epoch 730/1000 - Loss: 5.910280: 100%|██████████| 1/1 [00:00<00:00, 63.68it/s]\n",
      "Epoch 731/1000 - Loss: 5.910553: 100%|██████████| 1/1 [00:00<00:00, 62.34it/s]\n",
      "Epoch 732/1000 - Loss: 5.908996: 100%|██████████| 1/1 [00:00<00:00, 62.17it/s]\n",
      "Epoch 733/1000 - Loss: 5.912203: 100%|██████████| 1/1 [00:00<00:00, 64.01it/s]\n",
      "Epoch 734/1000 - Loss: 5.909177: 100%|██████████| 1/1 [00:00<00:00, 63.73it/s]\n",
      "Epoch 735/1000 - Loss: 5.911645: 100%|██████████| 1/1 [00:00<00:00, 63.65it/s]\n",
      "Epoch 736/1000 - Loss: 5.908994: 100%|██████████| 1/1 [00:00<00:00, 61.31it/s]\n",
      "Epoch 737/1000 - Loss: 5.906549: 100%|██████████| 1/1 [00:00<00:00, 60.85it/s]\n",
      "Epoch 738/1000 - Loss: 5.911800: 100%|██████████| 1/1 [00:00<00:00, 61.37it/s]\n",
      "Epoch 739/1000 - Loss: 5.910988: 100%|██████████| 1/1 [00:00<00:00, 60.85it/s]\n",
      "Epoch 740/1000 - Loss: 5.913155: 100%|██████████| 1/1 [00:00<00:00, 66.01it/s]\n",
      "Epoch 741/1000 - Loss: 5.908864: 100%|██████████| 1/1 [00:00<00:00, 66.00it/s]\n",
      "Epoch 742/1000 - Loss: 5.903955: 100%|██████████| 1/1 [00:00<00:00, 63.21it/s]\n",
      "Epoch 743/1000 - Loss: 5.911572: 100%|██████████| 1/1 [00:00<00:00, 57.54it/s]\n",
      "Epoch 744/1000 - Loss: 5.909400: 100%|██████████| 1/1 [00:00<00:00, 64.68it/s]\n",
      "Epoch 745/1000 - Loss: 5.904443: 100%|██████████| 1/1 [00:00<00:00, 65.27it/s]\n",
      "Epoch 746/1000 - Loss: 5.909011: 100%|██████████| 1/1 [00:00<00:00, 63.23it/s]\n",
      "Epoch 747/1000 - Loss: 5.906909: 100%|██████████| 1/1 [00:00<00:00, 69.62it/s]\n",
      "Epoch 748/1000 - Loss: 5.905030: 100%|██████████| 1/1 [00:00<00:00, 68.25it/s]\n",
      "Epoch 749/1000 - Loss: 5.911688: 100%|██████████| 1/1 [00:00<00:00, 62.87it/s]\n",
      "Epoch 750/1000 - Loss: 5.903797: 100%|██████████| 1/1 [00:00<00:00, 63.86it/s]\n",
      "Epoch 751/1000 - Loss: 5.908355: 100%|██████████| 1/1 [00:00<00:00, 65.17it/s]\n",
      "Epoch 752/1000 - Loss: 5.910323: 100%|██████████| 1/1 [00:00<00:00, 59.00it/s]\n",
      "Epoch 753/1000 - Loss: 5.908468: 100%|██████████| 1/1 [00:00<00:00, 65.76it/s]\n",
      "Epoch 754/1000 - Loss: 5.908592: 100%|██████████| 1/1 [00:00<00:00, 65.07it/s]\n",
      "Epoch 755/1000 - Loss: 5.907803: 100%|██████████| 1/1 [00:00<00:00, 60.90it/s]\n",
      "Epoch 756/1000 - Loss: 5.906099: 100%|██████████| 1/1 [00:00<00:00, 53.05it/s]\n",
      "Epoch 757/1000 - Loss: 5.905615: 100%|██████████| 1/1 [00:00<00:00, 57.02it/s]\n",
      "Epoch 758/1000 - Loss: 5.908842: 100%|██████████| 1/1 [00:00<00:00, 61.84it/s]\n",
      "Epoch 759/1000 - Loss: 5.910387: 100%|██████████| 1/1 [00:00<00:00, 63.22it/s]\n",
      "Epoch 760/1000 - Loss: 5.905771: 100%|██████████| 1/1 [00:00<00:00, 64.77it/s]\n",
      "Epoch 761/1000 - Loss: 5.908858: 100%|██████████| 1/1 [00:00<00:00, 60.76it/s]\n",
      "Epoch 762/1000 - Loss: 5.904223: 100%|██████████| 1/1 [00:00<00:00, 59.03it/s]\n",
      "Epoch 763/1000 - Loss: 5.897423: 100%|██████████| 1/1 [00:00<00:00, 61.51it/s]\n",
      "Epoch 764/1000 - Loss: 5.908334: 100%|██████████| 1/1 [00:00<00:00, 61.72it/s]\n",
      "Epoch 765/1000 - Loss: 5.906350: 100%|██████████| 1/1 [00:00<00:00, 60.99it/s]\n",
      "Epoch 766/1000 - Loss: 5.899477: 100%|██████████| 1/1 [00:00<00:00, 65.63it/s]\n",
      "Epoch 767/1000 - Loss: 5.905160: 100%|██████████| 1/1 [00:00<00:00, 66.88it/s]\n",
      "Epoch 768/1000 - Loss: 5.904634: 100%|██████████| 1/1 [00:00<00:00, 60.73it/s]\n",
      "Epoch 769/1000 - Loss: 5.900666: 100%|██████████| 1/1 [00:00<00:00, 62.96it/s]\n",
      "Epoch 770/1000 - Loss: 5.902811: 100%|██████████| 1/1 [00:00<00:00, 64.56it/s]\n",
      "Epoch 771/1000 - Loss: 5.905443: 100%|██████████| 1/1 [00:00<00:00, 62.20it/s]\n",
      "Epoch 772/1000 - Loss: 5.897090: 100%|██████████| 1/1 [00:00<00:00, 64.31it/s]\n",
      "Epoch 773/1000 - Loss: 5.897128: 100%|██████████| 1/1 [00:00<00:00, 60.92it/s]\n",
      "Epoch 774/1000 - Loss: 5.905480: 100%|██████████| 1/1 [00:00<00:00, 63.00it/s]\n",
      "Epoch 775/1000 - Loss: 5.902156: 100%|██████████| 1/1 [00:00<00:00, 63.89it/s]\n",
      "Epoch 776/1000 - Loss: 5.903765: 100%|██████████| 1/1 [00:00<00:00, 61.50it/s]\n",
      "Epoch 777/1000 - Loss: 5.896738: 100%|██████████| 1/1 [00:00<00:00, 58.49it/s]\n",
      "Epoch 778/1000 - Loss: 5.900767: 100%|██████████| 1/1 [00:00<00:00, 62.10it/s]\n",
      "Epoch 779/1000 - Loss: 5.892931: 100%|██████████| 1/1 [00:00<00:00, 60.20it/s]\n",
      "Epoch 780/1000 - Loss: 5.896723: 100%|██████████| 1/1 [00:00<00:00, 61.40it/s]\n",
      "Epoch 781/1000 - Loss: 5.904418: 100%|██████████| 1/1 [00:00<00:00, 60.36it/s]\n",
      "Epoch 782/1000 - Loss: 5.900252: 100%|██████████| 1/1 [00:00<00:00, 59.78it/s]\n",
      "Epoch 783/1000 - Loss: 5.903513: 100%|██████████| 1/1 [00:00<00:00, 62.96it/s]\n",
      "Epoch 784/1000 - Loss: 5.903698: 100%|██████████| 1/1 [00:00<00:00, 57.97it/s]\n",
      "Epoch 785/1000 - Loss: 5.901203: 100%|██████████| 1/1 [00:00<00:00, 59.64it/s]\n",
      "Epoch 786/1000 - Loss: 5.899408: 100%|██████████| 1/1 [00:00<00:00, 62.98it/s]\n",
      "Epoch 787/1000 - Loss: 5.895883: 100%|██████████| 1/1 [00:00<00:00, 65.67it/s]\n",
      "Epoch 788/1000 - Loss: 5.902730: 100%|██████████| 1/1 [00:00<00:00, 63.67it/s]\n",
      "Epoch 789/1000 - Loss: 5.897367: 100%|██████████| 1/1 [00:00<00:00, 61.14it/s]\n",
      "Epoch 790/1000 - Loss: 5.896921: 100%|██████████| 1/1 [00:00<00:00, 62.49it/s]\n",
      "Epoch 791/1000 - Loss: 5.900539: 100%|██████████| 1/1 [00:00<00:00, 63.31it/s]\n",
      "Epoch 792/1000 - Loss: 5.901464: 100%|██████████| 1/1 [00:00<00:00, 61.20it/s]\n",
      "Epoch 793/1000 - Loss: 5.899177: 100%|██████████| 1/1 [00:00<00:00, 71.22it/s]\n",
      "Epoch 794/1000 - Loss: 5.890892: 100%|██████████| 1/1 [00:00<00:00, 70.24it/s]\n",
      "Epoch 795/1000 - Loss: 5.889799: 100%|██████████| 1/1 [00:00<00:00, 69.29it/s]\n",
      "Epoch 796/1000 - Loss: 5.891139: 100%|██████████| 1/1 [00:00<00:00, 69.02it/s]\n",
      "Epoch 797/1000 - Loss: 5.898325: 100%|██████████| 1/1 [00:00<00:00, 59.76it/s]\n",
      "Epoch 798/1000 - Loss: 5.870686: 100%|██████████| 1/1 [00:00<00:00, 65.09it/s]\n",
      "Epoch 799/1000 - Loss: 5.876237: 100%|██████████| 1/1 [00:00<00:00, 67.42it/s]\n",
      "Epoch 800/1000 - Loss: 5.898138: 100%|██████████| 1/1 [00:00<00:00, 66.50it/s]\n",
      "Epoch 801/1000 - Loss: 5.880005: 100%|██████████| 1/1 [00:00<00:00, 64.37it/s]\n",
      "Epoch 802/1000 - Loss: 5.885170: 100%|██████████| 1/1 [00:00<00:00, 65.36it/s]\n",
      "Epoch 803/1000 - Loss: 5.860826: 100%|██████████| 1/1 [00:00<00:00, 61.98it/s]\n",
      "Epoch 804/1000 - Loss: 5.888644: 100%|██████████| 1/1 [00:00<00:00, 61.28it/s]\n",
      "Epoch 805/1000 - Loss: 5.897613: 100%|██████████| 1/1 [00:00<00:00, 62.98it/s]\n",
      "Epoch 806/1000 - Loss: 5.868488: 100%|██████████| 1/1 [00:00<00:00, 61.33it/s]\n",
      "Epoch 807/1000 - Loss: 5.870697: 100%|██████████| 1/1 [00:00<00:00, 61.17it/s]\n",
      "Epoch 808/1000 - Loss: 5.851279: 100%|██████████| 1/1 [00:00<00:00, 61.87it/s]\n",
      "Epoch 809/1000 - Loss: 5.860047: 100%|██████████| 1/1 [00:00<00:00, 61.38it/s]\n",
      "Epoch 810/1000 - Loss: 5.864304: 100%|██████████| 1/1 [00:00<00:00, 55.90it/s]\n",
      "Epoch 811/1000 - Loss: 5.860353: 100%|██████████| 1/1 [00:00<00:00, 61.87it/s]\n",
      "Epoch 812/1000 - Loss: 5.833964: 100%|██████████| 1/1 [00:00<00:00, 60.90it/s]\n",
      "Epoch 813/1000 - Loss: 5.832351: 100%|██████████| 1/1 [00:00<00:00, 62.66it/s]\n",
      "Epoch 814/1000 - Loss: 5.796191: 100%|██████████| 1/1 [00:00<00:00, 57.03it/s]\n",
      "Epoch 815/1000 - Loss: 5.800381: 100%|██████████| 1/1 [00:00<00:00, 64.86it/s]\n",
      "Epoch 816/1000 - Loss: 5.763655: 100%|██████████| 1/1 [00:00<00:00, 63.44it/s]\n",
      "Epoch 817/1000 - Loss: 5.728994: 100%|██████████| 1/1 [00:00<00:00, 62.32it/s]\n",
      "Epoch 818/1000 - Loss: 5.741498: 100%|██████████| 1/1 [00:00<00:00, 64.32it/s]\n",
      "Epoch 819/1000 - Loss: 5.679986: 100%|██████████| 1/1 [00:00<00:00, 60.73it/s]\n",
      "Epoch 820/1000 - Loss: 5.723527: 100%|██████████| 1/1 [00:00<00:00, 64.25it/s]\n",
      "Epoch 821/1000 - Loss: 5.797035: 100%|██████████| 1/1 [00:00<00:00, 61.17it/s]\n",
      "Epoch 822/1000 - Loss: 5.802957: 100%|██████████| 1/1 [00:00<00:00, 69.15it/s]\n",
      "Epoch 823/1000 - Loss: 5.806700: 100%|██████████| 1/1 [00:00<00:00, 70.40it/s]\n",
      "Epoch 824/1000 - Loss: 5.802518: 100%|██████████| 1/1 [00:00<00:00, 70.62it/s]\n",
      "Epoch 825/1000 - Loss: 5.757905: 100%|██████████| 1/1 [00:00<00:00, 70.33it/s]\n",
      "Epoch 826/1000 - Loss: 5.695360: 100%|██████████| 1/1 [00:00<00:00, 70.27it/s]\n",
      "Epoch 827/1000 - Loss: 5.720986: 100%|██████████| 1/1 [00:00<00:00, 70.30it/s]\n",
      "Epoch 828/1000 - Loss: 5.653696: 100%|██████████| 1/1 [00:00<00:00, 68.84it/s]\n",
      "Epoch 829/1000 - Loss: 5.604629: 100%|██████████| 1/1 [00:00<00:00, 68.88it/s]\n",
      "Epoch 830/1000 - Loss: 5.557197: 100%|██████████| 1/1 [00:00<00:00, 61.58it/s]\n",
      "Epoch 831/1000 - Loss: 5.472748: 100%|██████████| 1/1 [00:00<00:00, 71.04it/s]\n",
      "Epoch 832/1000 - Loss: 5.572135: 100%|██████████| 1/1 [00:00<00:00, 68.03it/s]\n",
      "Epoch 833/1000 - Loss: 5.523191: 100%|██████████| 1/1 [00:00<00:00, 69.91it/s]\n",
      "Epoch 834/1000 - Loss: 5.459341: 100%|██████████| 1/1 [00:00<00:00, 69.73it/s]\n",
      "Epoch 835/1000 - Loss: 5.446177: 100%|██████████| 1/1 [00:00<00:00, 70.36it/s]\n",
      "Epoch 836/1000 - Loss: 5.421034: 100%|██████████| 1/1 [00:00<00:00, 65.11it/s]\n",
      "Epoch 837/1000 - Loss: 5.368664: 100%|██████████| 1/1 [00:00<00:00, 62.37it/s]\n",
      "Epoch 838/1000 - Loss: 5.437546: 100%|██████████| 1/1 [00:00<00:00, 61.88it/s]\n",
      "Epoch 839/1000 - Loss: 5.399754: 100%|██████████| 1/1 [00:00<00:00, 60.99it/s]\n",
      "Epoch 840/1000 - Loss: 5.368458: 100%|██████████| 1/1 [00:00<00:00, 61.71it/s]\n",
      "Epoch 841/1000 - Loss: 5.373030: 100%|██████████| 1/1 [00:00<00:00, 59.99it/s]\n",
      "Epoch 842/1000 - Loss: 5.312025: 100%|██████████| 1/1 [00:00<00:00, 60.77it/s]\n",
      "Epoch 843/1000 - Loss: 5.296376: 100%|██████████| 1/1 [00:00<00:00, 50.76it/s]\n",
      "Epoch 844/1000 - Loss: 5.312785: 100%|██████████| 1/1 [00:00<00:00, 62.00it/s]\n",
      "Epoch 845/1000 - Loss: 5.301436: 100%|██████████| 1/1 [00:00<00:00, 62.54it/s]\n",
      "Epoch 846/1000 - Loss: 5.281698: 100%|██████████| 1/1 [00:00<00:00, 57.48it/s]\n",
      "Epoch 847/1000 - Loss: 5.265156: 100%|██████████| 1/1 [00:00<00:00, 63.30it/s]\n",
      "Epoch 848/1000 - Loss: 5.263079: 100%|██████████| 1/1 [00:00<00:00, 62.17it/s]\n",
      "Epoch 849/1000 - Loss: 5.238116: 100%|██████████| 1/1 [00:00<00:00, 63.56it/s]\n",
      "Epoch 850/1000 - Loss: 5.235884: 100%|██████████| 1/1 [00:00<00:00, 61.19it/s]\n",
      "Epoch 851/1000 - Loss: 5.261810: 100%|██████████| 1/1 [00:00<00:00, 59.27it/s]\n",
      "Epoch 852/1000 - Loss: 5.261004: 100%|██████████| 1/1 [00:00<00:00, 61.13it/s]\n",
      "Epoch 853/1000 - Loss: 5.226107: 100%|██████████| 1/1 [00:00<00:00, 61.29it/s]\n",
      "Epoch 854/1000 - Loss: 5.224121: 100%|██████████| 1/1 [00:00<00:00, 63.94it/s]\n",
      "Epoch 855/1000 - Loss: 5.225879: 100%|██████████| 1/1 [00:00<00:00, 66.36it/s]\n",
      "Epoch 856/1000 - Loss: 5.213523: 100%|██████████| 1/1 [00:00<00:00, 63.20it/s]\n",
      "Epoch 857/1000 - Loss: 5.223851: 100%|██████████| 1/1 [00:00<00:00, 64.58it/s]\n",
      "Epoch 858/1000 - Loss: 5.222442: 100%|██████████| 1/1 [00:00<00:00, 55.78it/s]\n",
      "Epoch 859/1000 - Loss: 5.223107: 100%|██████████| 1/1 [00:00<00:00, 66.44it/s]\n",
      "Epoch 860/1000 - Loss: 5.219798: 100%|██████████| 1/1 [00:00<00:00, 58.98it/s]\n",
      "Epoch 861/1000 - Loss: 5.207353: 100%|██████████| 1/1 [00:00<00:00, 63.30it/s]\n",
      "Epoch 862/1000 - Loss: 5.184976: 100%|██████████| 1/1 [00:00<00:00, 63.71it/s]\n",
      "Epoch 863/1000 - Loss: 5.196635: 100%|██████████| 1/1 [00:00<00:00, 61.69it/s]\n",
      "Epoch 864/1000 - Loss: 5.228191: 100%|██████████| 1/1 [00:00<00:00, 61.31it/s]\n",
      "Epoch 865/1000 - Loss: 5.184205: 100%|██████████| 1/1 [00:00<00:00, 58.79it/s]\n",
      "Epoch 866/1000 - Loss: 5.224987: 100%|██████████| 1/1 [00:00<00:00, 63.89it/s]\n",
      "Epoch 867/1000 - Loss: 5.207812: 100%|██████████| 1/1 [00:00<00:00, 60.12it/s]\n",
      "Epoch 868/1000 - Loss: 5.237691: 100%|██████████| 1/1 [00:00<00:00, 64.26it/s]\n",
      "Epoch 869/1000 - Loss: 5.351939: 100%|██████████| 1/1 [00:00<00:00, 59.73it/s]\n",
      "Epoch 870/1000 - Loss: 5.374005: 100%|██████████| 1/1 [00:00<00:00, 62.53it/s]\n",
      "Epoch 871/1000 - Loss: 5.320346: 100%|██████████| 1/1 [00:00<00:00, 64.67it/s]\n",
      "Epoch 872/1000 - Loss: 5.199197: 100%|██████████| 1/1 [00:00<00:00, 61.36it/s]\n",
      "Epoch 873/1000 - Loss: 5.357236: 100%|██████████| 1/1 [00:00<00:00, 62.53it/s]\n",
      "Epoch 874/1000 - Loss: 5.284647: 100%|██████████| 1/1 [00:00<00:00, 64.12it/s]\n",
      "Epoch 875/1000 - Loss: 5.373583: 100%|██████████| 1/1 [00:00<00:00, 61.93it/s]\n",
      "Epoch 876/1000 - Loss: 5.296017: 100%|██████████| 1/1 [00:00<00:00, 61.57it/s]\n",
      "Epoch 877/1000 - Loss: 5.339847: 100%|██████████| 1/1 [00:00<00:00, 60.80it/s]\n",
      "Epoch 878/1000 - Loss: 5.396938: 100%|██████████| 1/1 [00:00<00:00, 60.86it/s]\n",
      "Epoch 879/1000 - Loss: 5.349954: 100%|██████████| 1/1 [00:00<00:00, 60.98it/s]\n",
      "Epoch 880/1000 - Loss: 5.344913: 100%|██████████| 1/1 [00:00<00:00, 59.74it/s]\n",
      "Epoch 881/1000 - Loss: 5.330542: 100%|██████████| 1/1 [00:00<00:00, 61.59it/s]\n",
      "Epoch 882/1000 - Loss: 5.285961: 100%|██████████| 1/1 [00:00<00:00, 58.95it/s]\n",
      "Epoch 883/1000 - Loss: 5.204000: 100%|██████████| 1/1 [00:00<00:00, 64.13it/s]\n",
      "Epoch 884/1000 - Loss: 5.325098: 100%|██████████| 1/1 [00:00<00:00, 59.07it/s]\n",
      "Epoch 885/1000 - Loss: 5.216368: 100%|██████████| 1/1 [00:00<00:00, 59.89it/s]\n",
      "Epoch 886/1000 - Loss: 5.208608: 100%|██████████| 1/1 [00:00<00:00, 60.57it/s]\n",
      "Epoch 887/1000 - Loss: 5.224423: 100%|██████████| 1/1 [00:00<00:00, 60.29it/s]\n",
      "Epoch 888/1000 - Loss: 5.241866: 100%|██████████| 1/1 [00:00<00:00, 59.03it/s]\n",
      "Epoch 889/1000 - Loss: 5.247333: 100%|██████████| 1/1 [00:00<00:00, 59.99it/s]\n",
      "Epoch 890/1000 - Loss: 5.254710: 100%|██████████| 1/1 [00:00<00:00, 56.69it/s]\n",
      "Epoch 891/1000 - Loss: 5.250291: 100%|██████████| 1/1 [00:00<00:00, 57.97it/s]\n",
      "Epoch 892/1000 - Loss: 5.211841: 100%|██████████| 1/1 [00:00<00:00, 62.01it/s]\n",
      "Epoch 893/1000 - Loss: 5.199871: 100%|██████████| 1/1 [00:00<00:00, 59.25it/s]\n",
      "Epoch 894/1000 - Loss: 5.222795: 100%|██████████| 1/1 [00:00<00:00, 58.24it/s]\n",
      "Epoch 895/1000 - Loss: 5.214731: 100%|██████████| 1/1 [00:00<00:00, 60.39it/s]\n",
      "Epoch 896/1000 - Loss: 5.218655: 100%|██████████| 1/1 [00:00<00:00, 58.66it/s]\n",
      "Epoch 897/1000 - Loss: 5.223184: 100%|██████████| 1/1 [00:00<00:00, 67.63it/s]\n",
      "Epoch 898/1000 - Loss: 5.193560: 100%|██████████| 1/1 [00:00<00:00, 59.75it/s]\n",
      "Epoch 899/1000 - Loss: 5.214946: 100%|██████████| 1/1 [00:00<00:00, 61.82it/s]\n",
      "Epoch 900/1000 - Loss: 5.203941: 100%|██████████| 1/1 [00:00<00:00, 57.87it/s]\n",
      "Epoch 901/1000 - Loss: 5.202346: 100%|██████████| 1/1 [00:00<00:00, 59.71it/s]\n",
      "Epoch 902/1000 - Loss: 5.198234: 100%|██████████| 1/1 [00:00<00:00, 56.93it/s]\n",
      "Epoch 903/1000 - Loss: 5.177190: 100%|██████████| 1/1 [00:00<00:00, 59.25it/s]\n",
      "Epoch 904/1000 - Loss: 5.189681: 100%|██████████| 1/1 [00:00<00:00, 59.82it/s]\n",
      "Epoch 905/1000 - Loss: 5.200937: 100%|██████████| 1/1 [00:00<00:00, 56.98it/s]\n",
      "Epoch 906/1000 - Loss: 5.191631: 100%|██████████| 1/1 [00:00<00:00, 65.10it/s]\n",
      "Epoch 907/1000 - Loss: 5.186218: 100%|██████████| 1/1 [00:00<00:00, 64.31it/s]\n",
      "Epoch 908/1000 - Loss: 5.194142: 100%|██████████| 1/1 [00:00<00:00, 57.74it/s]\n",
      "Epoch 909/1000 - Loss: 5.184784: 100%|██████████| 1/1 [00:00<00:00, 62.13it/s]\n",
      "Epoch 910/1000 - Loss: 5.166494: 100%|██████████| 1/1 [00:00<00:00, 62.93it/s]\n",
      "Epoch 911/1000 - Loss: 5.182742: 100%|██████████| 1/1 [00:00<00:00, 63.14it/s]\n",
      "Epoch 912/1000 - Loss: 5.191453: 100%|██████████| 1/1 [00:00<00:00, 62.87it/s]\n",
      "Epoch 913/1000 - Loss: 5.196517: 100%|██████████| 1/1 [00:00<00:00, 57.29it/s]\n",
      "Epoch 914/1000 - Loss: 5.190429: 100%|██████████| 1/1 [00:00<00:00, 62.07it/s]\n",
      "Epoch 915/1000 - Loss: 5.182284: 100%|██████████| 1/1 [00:00<00:00, 58.17it/s]\n",
      "Epoch 916/1000 - Loss: 5.202473: 100%|██████████| 1/1 [00:00<00:00, 61.30it/s]\n",
      "Epoch 917/1000 - Loss: 5.183177: 100%|██████████| 1/1 [00:00<00:00, 61.58it/s]\n",
      "Epoch 918/1000 - Loss: 5.189478: 100%|██████████| 1/1 [00:00<00:00, 55.58it/s]\n",
      "Epoch 919/1000 - Loss: 5.180884: 100%|██████████| 1/1 [00:00<00:00, 59.00it/s]\n",
      "Epoch 920/1000 - Loss: 5.178818: 100%|██████████| 1/1 [00:00<00:00, 60.66it/s]\n",
      "Epoch 921/1000 - Loss: 5.200392: 100%|██████████| 1/1 [00:00<00:00, 57.95it/s]\n",
      "Epoch 922/1000 - Loss: 5.181763: 100%|██████████| 1/1 [00:00<00:00, 56.06it/s]\n",
      "Epoch 923/1000 - Loss: 5.189735: 100%|██████████| 1/1 [00:00<00:00, 54.95it/s]\n",
      "Epoch 924/1000 - Loss: 5.181502: 100%|██████████| 1/1 [00:00<00:00, 53.87it/s]\n",
      "Epoch 925/1000 - Loss: 5.179261: 100%|██████████| 1/1 [00:00<00:00, 54.58it/s]\n",
      "Epoch 926/1000 - Loss: 5.180223: 100%|██████████| 1/1 [00:00<00:00, 58.50it/s]\n",
      "Epoch 927/1000 - Loss: 5.186708: 100%|██████████| 1/1 [00:00<00:00, 58.72it/s]\n",
      "Epoch 928/1000 - Loss: 5.176443: 100%|██████████| 1/1 [00:00<00:00, 68.67it/s]\n",
      "Epoch 929/1000 - Loss: 5.177678: 100%|██████████| 1/1 [00:00<00:00, 67.93it/s]\n",
      "Epoch 930/1000 - Loss: 5.178385: 100%|██████████| 1/1 [00:00<00:00, 55.52it/s]\n",
      "Epoch 931/1000 - Loss: 5.188820: 100%|██████████| 1/1 [00:00<00:00, 58.25it/s]\n",
      "Epoch 932/1000 - Loss: 5.181080: 100%|██████████| 1/1 [00:00<00:00, 59.80it/s]\n",
      "Epoch 933/1000 - Loss: 5.199230: 100%|██████████| 1/1 [00:00<00:00, 70.33it/s]\n",
      "Epoch 934/1000 - Loss: 5.186646: 100%|██████████| 1/1 [00:00<00:00, 58.61it/s]\n",
      "Epoch 935/1000 - Loss: 5.176919: 100%|██████████| 1/1 [00:00<00:00, 62.32it/s]\n",
      "Epoch 936/1000 - Loss: 5.174574: 100%|██████████| 1/1 [00:00<00:00, 59.60it/s]\n",
      "Epoch 937/1000 - Loss: 5.180077: 100%|██████████| 1/1 [00:00<00:00, 57.10it/s]\n",
      "Epoch 938/1000 - Loss: 5.175406: 100%|██████████| 1/1 [00:00<00:00, 55.24it/s]\n",
      "Epoch 939/1000 - Loss: 5.182958: 100%|██████████| 1/1 [00:00<00:00, 56.30it/s]\n",
      "Epoch 940/1000 - Loss: 5.170970: 100%|██████████| 1/1 [00:00<00:00, 57.33it/s]\n",
      "Epoch 941/1000 - Loss: 5.177861: 100%|██████████| 1/1 [00:00<00:00, 54.21it/s]\n",
      "Epoch 942/1000 - Loss: 5.173904: 100%|██████████| 1/1 [00:00<00:00, 52.15it/s]\n",
      "Epoch 943/1000 - Loss: 5.166309: 100%|██████████| 1/1 [00:00<00:00, 55.03it/s]\n",
      "Epoch 944/1000 - Loss: 5.156503: 100%|██████████| 1/1 [00:00<00:00, 58.03it/s]\n",
      "Epoch 945/1000 - Loss: 5.180417: 100%|██████████| 1/1 [00:00<00:00, 61.20it/s]\n",
      "Epoch 946/1000 - Loss: 5.172093: 100%|██████████| 1/1 [00:00<00:00, 59.42it/s]\n",
      "Epoch 947/1000 - Loss: 5.172389: 100%|██████████| 1/1 [00:00<00:00, 56.93it/s]\n",
      "Epoch 948/1000 - Loss: 5.174621: 100%|██████████| 1/1 [00:00<00:00, 59.36it/s]\n",
      "Epoch 949/1000 - Loss: 5.180882: 100%|██████████| 1/1 [00:00<00:00, 63.58it/s]\n",
      "Epoch 950/1000 - Loss: 5.175499: 100%|██████████| 1/1 [00:00<00:00, 63.51it/s]\n",
      "Epoch 951/1000 - Loss: 5.180164: 100%|██████████| 1/1 [00:00<00:00, 63.80it/s]\n",
      "Epoch 952/1000 - Loss: 5.176512: 100%|██████████| 1/1 [00:00<00:00, 57.40it/s]\n",
      "Epoch 953/1000 - Loss: 5.171395: 100%|██████████| 1/1 [00:00<00:00, 58.24it/s]\n",
      "Epoch 954/1000 - Loss: 5.181201: 100%|██████████| 1/1 [00:00<00:00, 60.38it/s]\n",
      "Epoch 955/1000 - Loss: 5.176167: 100%|██████████| 1/1 [00:00<00:00, 61.03it/s]\n",
      "Epoch 956/1000 - Loss: 5.173709: 100%|██████████| 1/1 [00:00<00:00, 60.19it/s]\n",
      "Epoch 957/1000 - Loss: 5.173341: 100%|██████████| 1/1 [00:00<00:00, 63.16it/s]\n",
      "Epoch 958/1000 - Loss: 5.169034: 100%|██████████| 1/1 [00:00<00:00, 51.23it/s]\n",
      "Epoch 959/1000 - Loss: 5.173547: 100%|██████████| 1/1 [00:00<00:00, 55.97it/s]\n",
      "Epoch 960/1000 - Loss: 5.168401: 100%|██████████| 1/1 [00:00<00:00, 54.55it/s]\n",
      "Epoch 961/1000 - Loss: 5.176418: 100%|██████████| 1/1 [00:00<00:00, 54.82it/s]\n",
      "Epoch 962/1000 - Loss: 5.174622: 100%|██████████| 1/1 [00:00<00:00, 57.11it/s]\n",
      "Epoch 963/1000 - Loss: 5.169266: 100%|██████████| 1/1 [00:00<00:00, 58.34it/s]\n",
      "Epoch 964/1000 - Loss: 5.178224: 100%|██████████| 1/1 [00:00<00:00, 58.80it/s]\n",
      "Epoch 965/1000 - Loss: 5.169441: 100%|██████████| 1/1 [00:00<00:00, 60.08it/s]\n",
      "Epoch 966/1000 - Loss: 5.165627: 100%|██████████| 1/1 [00:00<00:00, 61.57it/s]\n",
      "Epoch 967/1000 - Loss: 5.184557: 100%|██████████| 1/1 [00:00<00:00, 60.21it/s]\n",
      "Epoch 968/1000 - Loss: 5.161343: 100%|██████████| 1/1 [00:00<00:00, 60.48it/s]\n",
      "Epoch 969/1000 - Loss: 5.165051: 100%|██████████| 1/1 [00:00<00:00, 60.75it/s]\n",
      "Epoch 970/1000 - Loss: 5.161150: 100%|██████████| 1/1 [00:00<00:00, 65.88it/s]\n",
      "Epoch 971/1000 - Loss: 5.164968: 100%|██████████| 1/1 [00:00<00:00, 59.27it/s]\n",
      "Epoch 972/1000 - Loss: 5.173383: 100%|██████████| 1/1 [00:00<00:00, 63.87it/s]\n",
      "Epoch 973/1000 - Loss: 5.171596: 100%|██████████| 1/1 [00:00<00:00, 60.12it/s]\n",
      "Epoch 974/1000 - Loss: 5.170953: 100%|██████████| 1/1 [00:00<00:00, 59.41it/s]\n",
      "Epoch 975/1000 - Loss: 5.156955: 100%|██████████| 1/1 [00:00<00:00, 56.98it/s]\n",
      "Epoch 976/1000 - Loss: 5.167008: 100%|██████████| 1/1 [00:00<00:00, 62.79it/s]\n",
      "Epoch 977/1000 - Loss: 5.174965: 100%|██████████| 1/1 [00:00<00:00, 57.77it/s]\n",
      "Epoch 978/1000 - Loss: 5.159921: 100%|██████████| 1/1 [00:00<00:00, 59.85it/s]\n",
      "Epoch 979/1000 - Loss: 5.166612: 100%|██████████| 1/1 [00:00<00:00, 59.99it/s]\n",
      "Epoch 980/1000 - Loss: 5.158744: 100%|██████████| 1/1 [00:00<00:00, 56.59it/s]\n",
      "Epoch 981/1000 - Loss: 5.165849: 100%|██████████| 1/1 [00:00<00:00, 61.01it/s]\n",
      "Epoch 982/1000 - Loss: 5.165535: 100%|██████████| 1/1 [00:00<00:00, 59.85it/s]\n",
      "Epoch 983/1000 - Loss: 5.160854: 100%|██████████| 1/1 [00:00<00:00, 59.96it/s]\n",
      "Epoch 984/1000 - Loss: 5.169430: 100%|██████████| 1/1 [00:00<00:00, 61.55it/s]\n",
      "Epoch 985/1000 - Loss: 5.172140: 100%|██████████| 1/1 [00:00<00:00, 59.13it/s]\n",
      "Epoch 986/1000 - Loss: 5.170432: 100%|██████████| 1/1 [00:00<00:00, 58.79it/s]\n",
      "Epoch 987/1000 - Loss: 5.161038: 100%|██████████| 1/1 [00:00<00:00, 52.11it/s]\n",
      "Epoch 988/1000 - Loss: 5.158587: 100%|██████████| 1/1 [00:00<00:00, 55.07it/s]\n",
      "Epoch 989/1000 - Loss: 5.156293: 100%|██████████| 1/1 [00:00<00:00, 58.95it/s]\n",
      "Epoch 990/1000 - Loss: 5.149325: 100%|██████████| 1/1 [00:00<00:00, 52.70it/s]\n",
      "Epoch 991/1000 - Loss: 5.207417: 100%|██████████| 1/1 [00:00<00:00, 60.31it/s]\n",
      "Epoch 992/1000 - Loss: 5.166513: 100%|██████████| 1/1 [00:00<00:00, 60.18it/s]\n",
      "Epoch 993/1000 - Loss: 5.157434: 100%|██████████| 1/1 [00:00<00:00, 59.14it/s]\n",
      "Epoch 994/1000 - Loss: 5.193542: 100%|██████████| 1/1 [00:00<00:00, 51.10it/s]\n",
      "Epoch 995/1000 - Loss: 5.193081: 100%|██████████| 1/1 [00:00<00:00, 56.50it/s]\n",
      "Epoch 996/1000 - Loss: 5.179616: 100%|██████████| 1/1 [00:00<00:00, 59.63it/s]\n",
      "Epoch 997/1000 - Loss: 5.188038: 100%|██████████| 1/1 [00:00<00:00, 58.25it/s]\n",
      "Epoch 998/1000 - Loss: 5.171086: 100%|██████████| 1/1 [00:00<00:00, 56.63it/s]\n",
      "Epoch 999/1000 - Loss: 5.178452: 100%|██████████| 1/1 [00:00<00:00, 58.69it/s]\n",
      "Epoch 1000/1000 - Loss: 5.171940: 100%|██████████| 1/1 [00:00<00:00, 57.77it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "input_dim = data.X.shape[-1]\n",
    "encoder = EncoderProjectionHead(input_dim).to(device)\n",
    "epochs = 1000\n",
    "optimizer = optim.Adam(encoder.parameters(), lr=0.001)\n",
    "batch_size = dataloader.batch_size\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    epoch_loss = 0\n",
    "\n",
    "    with tqdm(dataloader, desc=f\"Epoch {epoch+1}/{epochs}\") as pbar:\n",
    "        for d in pbar:\n",
    "            d = d.to(device).view(-1, d.size()[-1])\n",
    "            optimizer.zero_grad()\n",
    "            z = encoder(d)\n",
    "            logits, labels = info_nce_loss(z, batch_size, device)\n",
    "            loss = criterion(logits, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            pbar.set_description(f\"Epoch {epoch+1}/{epochs} - Loss: {loss.item():.6f}\")\n",
    "\n",
    "    avg_loss = epoch_loss / len(dataloader)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(174, 512)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder.eval()\n",
    "with torch.no_grad():\n",
    "    feature_extractor = encoder.encoder\n",
    "    embedding = feature_extractor(torch.tensor(data.X).to(device)).cpu().numpy()\n",
    "embedding.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## We can use the embedding for downstream task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validated Accuracy: 0.5345\n",
      "Cross-validated AUROC: 0.5072\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "y = np.argmax(data.y, axis=1)\n",
    "logistic_cv = LogisticRegressionCV(Cs = 1, max_iter=1000, random_state=0)\n",
    "\n",
    "y_pred_cv = cross_val_predict(logistic_cv, embedding, y, cv=5, method=\"predict\")\n",
    "\n",
    "auroc = roc_auc_score(y, y_pred_cv)\n",
    "\n",
    "accuracy = accuracy_score(y, y_pred_cv)\n",
    "\n",
    "print(f\"Cross-validated Accuracy: {accuracy:.4f}\")\n",
    "print(f\"Cross-validated AUROC: {auroc:.4f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mic_aug",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
